diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp --- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp @@ -438,6 +438,20 @@ /// Index that keeps track of where to find a metadata record in the stream. std::vector GlobalMetadataBitPosIndex; + /// Cursor position of the start of the global decl attachments, to enable + /// loading using the index built for lazy loading, instead of forward + /// references. + uint64_t GlobalDeclAttachmentPos = 0; + +#ifndef NDEBUG + /// Sanity check that we end up parsing all of the global decl attachments. + unsigned NumGlobalDeclAttachSkipped = 0; + unsigned NumGlobalDeclAttachParsed = 0; +#endif + + /// Load the global decl attachments, using the index built for lazy loading. + Expected loadGlobalDeclAttachments(); + /// Populate the index above to enable lazily loading of metadata, and load /// the named metadata as well as the transitively referenced global /// Metadata. @@ -681,8 +695,10 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { IndexCursor = Stream; SmallVector Record; + GlobalDeclAttachmentPos = 0; // Get the abbrevs, and preload record positions to make them lazy-loadable. while (true) { + uint64_t SavedPos = IndexCursor.GetCurrentBitNo(); Expected MaybeEntry = IndexCursor.advanceSkippingSubblocks( BitstreamCursor::AF_DontPopBlockAtEnd); if (!MaybeEntry) @@ -817,25 +833,11 @@ break; } case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { - // FIXME: we need to do this early because we don't materialize global - // value explicitly. - if (Error Err = IndexCursor.JumpToBit(CurrentPos)) - return std::move(Err); - Record.clear(); - if (Expected MaybeRecord = - IndexCursor.readRecord(Entry.ID, Record)) - ; - else - return MaybeRecord.takeError(); - if (Record.size() % 2 == 0) - return error("Invalid record"); - unsigned ValueID = Record[0]; - if (ValueID >= ValueList.size()) - return error("Invalid record"); - if (auto *GO = dyn_cast(ValueList[ValueID])) - if (Error Err = parseGlobalObjectAttachment( - *GO, ArrayRef(Record).slice(1))) - return std::move(Err); + if (!GlobalDeclAttachmentPos) + GlobalDeclAttachmentPos = SavedPos; +#ifndef NDEBUG + NumGlobalDeclAttachSkipped++; +#endif break; } case bitc::METADATA_KIND: @@ -885,6 +887,80 @@ } } +// Load the global decl attachments after building the lazy loading index. +// We don't load them "lazily" - all global decl attachments must be +// parsed since they aren't materialized on demand. However, by delaying +// their parsing until after the index is created, we can use the index +// instead of creating temporaries. +Expected MetadataLoader::MetadataLoaderImpl::loadGlobalDeclAttachments() { + // Nothing to do if we didn't find any of these metadata records. + if (!GlobalDeclAttachmentPos) + return true; + IndexCursor = Stream; + SmallVector Record; + // Jump to the position before the first global decl attachment, so we can + // scan for the first BitstreamEntry record. + if (Error Err = IndexCursor.JumpToBit(GlobalDeclAttachmentPos)) + return std::move(Err); + while (true) { + Expected MaybeEntry = IndexCursor.advanceSkippingSubblocks( + BitstreamCursor::AF_DontPopBlockAtEnd); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + + switch (Entry.Kind) { + case BitstreamEntry::SubBlock: // Handled for us already. + case BitstreamEntry::Error: + return error("Malformed block"); + case BitstreamEntry::EndBlock: { + // Sanity check that we parsed them all. + assert(NumGlobalDeclAttachSkipped == NumGlobalDeclAttachParsed); + return true; + } + case BitstreamEntry::Record: + break; + } + uint64_t CurrentPos = IndexCursor.GetCurrentBitNo(); + Expected MaybeCode = IndexCursor.skipRecord(Entry.ID); + if (!MaybeCode) + return MaybeCode.takeError(); + unsigned Code = MaybeCode.get(); + switch (Code) { + default: + // Anything other than a global decl attachment signals the end of + // these records. sanity check that we parsed them all. + assert(NumGlobalDeclAttachSkipped == NumGlobalDeclAttachParsed); + return true; + case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { +#ifndef NDEBUG + NumGlobalDeclAttachParsed++; +#endif + // FIXME: we need to do this early because we don't materialize global + // value explicitly. + if (Error Err = IndexCursor.JumpToBit(CurrentPos)) + return std::move(Err); + Record.clear(); + if (Expected MaybeRecord = + IndexCursor.readRecord(Entry.ID, Record)) + ; + else + return MaybeRecord.takeError(); + if (Record.size() % 2 == 0) + return error("Invalid record"); + unsigned ValueID = Record[0]; + if (ValueID >= ValueList.size()) + return error("Invalid record"); + if (auto *GO = dyn_cast(ValueList[ValueID])) + if (Error Err = parseGlobalObjectAttachment( + *GO, ArrayRef(Record).slice(1))) + return std::move(Err); + break; + } + } + } +} + /// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing /// module level metadata. Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { @@ -914,6 +990,14 @@ MetadataList.resize(MDStringRef.size() + GlobalMetadataBitPosIndex.size()); + // Now that we have built the index, load the global decl attachments + // that were deferred during that process. This avoids creating + // temporaries. + SuccessOrErr = loadGlobalDeclAttachments(); + if (!SuccessOrErr) + return SuccessOrErr.takeError(); + assert(SuccessOrErr.get()); + // Reading the named metadata created forward references and/or // placeholders, that we flush here. resolveForwardRefsAndPlaceholders(Placeholders); @@ -2021,7 +2105,8 @@ auto K = MDKindMap.find(Record[I]); if (K == MDKindMap.end()) return error("Invalid ID"); - MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Record[I + 1]); + MDNode *MD = + dyn_cast_or_null(getMetadataFwdRefOrLoad(Record[I + 1])); if (!MD) return error("Invalid metadata attachment: expect fwd ref to MDNode"); GO.addMetadata(K->second, *MD);