Index: lld/trunk/COFF/Config.h =================================================================== --- lld/trunk/COFF/Config.h +++ lld/trunk/COFF/Config.h @@ -99,6 +99,7 @@ bool DebugGHashes = false; bool ShowTiming = false; unsigned DebugTypes = static_cast(DebugType::None); + std::vector NatvisFiles; llvm::SmallString<128> PDBPath; std::vector Argv; Index: lld/trunk/COFF/Driver.cpp =================================================================== --- lld/trunk/COFF/Driver.cpp +++ lld/trunk/COFF/Driver.cpp @@ -933,9 +933,12 @@ // Handle /pdb bool ShouldCreatePDB = Args.hasArg(OPT_debug, OPT_debug_ghash); - if (ShouldCreatePDB) + if (ShouldCreatePDB) { if (auto *Arg = Args.getLastArg(OPT_pdb)) Config->PDBPath = Arg->getValue(); + if (Args.hasArg(OPT_natvis)) + Config->NatvisFiles = Args.getAllArgValues(OPT_natvis); + } // Handle /noentry if (Args.hasArg(OPT_noentry)) { Index: lld/trunk/COFF/Options.td =================================================================== --- lld/trunk/COFF/Options.td +++ lld/trunk/COFF/Options.td @@ -45,6 +45,7 @@ def opt : P<"opt", "Control optimizations">; def order : P<"order", "Put functions in order">; def out : P<"out", "Path to file to write output">; +def natvis : P<"natvis", "Path to natvis file to embed in the PDB">; def pdb : P<"pdb", "PDB file path">; def section : P<"section", "Specify section attributes">; def stack : P<"stack", "Size of the stack">; @@ -162,7 +163,6 @@ def errorreport : QF<"errorreport">; def idlout : QF<"idlout">; def maxilksize : QF<"maxilksize">; -def natvis : QF<"natvis">; def pdbaltpath : QF<"pdbaltpath">; def tlbid : QF<"tlbid">; def tlbout : QF<"tlbout">; Index: lld/trunk/COFF/PDB.cpp =================================================================== --- lld/trunk/COFF/PDB.cpp +++ lld/trunk/COFF/PDB.cpp @@ -90,6 +90,9 @@ /// Emit the basic PDB structure: initial streams, headers, etc. void initialize(const llvm::codeview::DebugInfo &BuildId); + /// Add natvis files specified on the command line. + void addNatvisFiles(); + /// Link CodeView from each object file in the symbol table into the PDB. void addObjectsToPDB(); @@ -961,6 +964,18 @@ } } +void PDBLinker::addNatvisFiles() { + for (StringRef File : Config->NatvisFiles) { + ErrorOr> DataOrErr = + MemoryBuffer::getFile(File); + if (!DataOrErr) { + warn("Cannot open input file: " + File); + continue; + } + Builder.addInjectedSource(File, std::move(*DataOrErr)); + } +} + static void addCommonLinkerModuleSymbols(StringRef Path, pdb::DbiModuleDescriptorBuilder &Mod, BumpPtrAllocator &Allocator) { @@ -1041,6 +1056,7 @@ PDB.initialize(BuildId); PDB.addObjectsToPDB(); PDB.addSections(OutputSections, SectionTable); + PDB.addNatvisFiles(); ScopedTimer T2(DiskCommitTimer); PDB.commit(); Index: lld/trunk/test/COFF/Inputs/generic.yaml =================================================================== --- lld/trunk/test/COFF/Inputs/generic.yaml +++ lld/trunk/test/COFF/Inputs/generic.yaml @@ -0,0 +1,282 @@ +--- !COFF +header: + Machine: IMAGE_FILE_MACHINE_AMD64 + Characteristics: [ ] +sections: + - Name: .text + Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] + Alignment: 16 + SectionData: 4883EC1831C0C7442414000000004889542408894C24044883C418C3 + - Name: .data + Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ] + Alignment: 4 + SectionData: '' + - Name: .bss + Characteristics: [ IMAGE_SCN_CNT_UNINITIALIZED_DATA, IMAGE_SCN_MEM_READ, IMAGE_SCN_MEM_WRITE ] + Alignment: 4 + SectionData: '' + - Name: .xdata + Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ] + Alignment: 4 + SectionData: '0104010004220000' + - Name: .drectve + Characteristics: [ IMAGE_SCN_LNK_INFO, IMAGE_SCN_LNK_REMOVE ] + Alignment: 1 + SectionData: 202F44454641554C544C49423A6C6962636D742E6C6962202F44454641554C544C49423A6F6C646E616D65732E6C6962 + - Name: '.debug$S' + Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ] + Alignment: 4 + SectionData: 04000000F10000002F0000002D003C1101000000D0000700000000000000581B000000000000636C616E672076657273696F6E20372E302E30200000F1000000760000002A0047110000000000000000000000001C000000000000000000000003100000000000000000006D61696E000D003E117400000001006172676300120045114F0100000400000017000000000005000D003E110010000001006172677600120045114F01000008000000170000000000050002004F110000F20000002800000000000000000000001C00000000000000020000001C00000000000000020000001700000003000000F40000001800000001000000100139E9A066A1995A99DD01F5A392F26D7C0000F30000003000000000443A5C7372635C6C6C766D6275696C645C636C5C52656C656173655C7836345C67656E657269632E63707000000000 + Subsections: + - !Symbols + Records: + - Kind: S_COMPILE3 + Compile3Sym: + Flags: [ ] + Machine: X64 + FrontendMajor: 7 + FrontendMinor: 0 + FrontendBuild: 0 + FrontendQFE: 0 + BackendMajor: 7000 + BackendMinor: 0 + BackendBuild: 0 + BackendQFE: 0 + Version: 'clang version 7.0.0 ' + - !Symbols + Records: + - Kind: S_GPROC32_ID + ProcSym: + CodeSize: 28 + DbgStart: 0 + DbgEnd: 0 + FunctionType: 4099 + Flags: [ ] + DisplayName: main + - Kind: S_LOCAL + LocalSym: + Type: 116 + Flags: [ IsParameter ] + VarName: argc + - Kind: S_DEFRANGE_REGISTER_REL + DefRangeRegisterRelSym: + Register: 335 + Flags: 0 + BasePointerOffset: 4 + Range: + OffsetStart: 23 + ISectStart: 0 + Range: 5 + Gaps: + - Kind: S_LOCAL + LocalSym: + Type: 4096 + Flags: [ IsParameter ] + VarName: argv + - Kind: S_DEFRANGE_REGISTER_REL + DefRangeRegisterRelSym: + Register: 335 + Flags: 0 + BasePointerOffset: 8 + Range: + OffsetStart: 23 + ISectStart: 0 + Range: 5 + Gaps: + - Kind: S_PROC_ID_END + ScopeEndSym: + - !Lines + CodeSize: 28 + Flags: [ ] + RelocOffset: 0 + RelocSegment: 0 + Blocks: + - FileName: 'D:\src\llvmbuild\cl\Release\x64\generic.cpp' + Lines: + - Offset: 0 + LineStart: 2 + IsStatement: false + EndDelta: 0 + - Offset: 23 + LineStart: 3 + IsStatement: false + EndDelta: 0 + Columns: + - !FileChecksums + Checksums: + - FileName: 'D:\src\llvmbuild\cl\Release\x64\generic.cpp' + Kind: MD5 + Checksum: 39E9A066A1995A99DD01F5A392F26D7C + - !StringTable + Strings: + - 'D:\src\llvmbuild\cl\Release\x64\generic.cpp' + - '' + - '' + - '' + Relocations: + - VirtualAddress: 100 + SymbolName: main + Type: IMAGE_REL_AMD64_SECREL + - VirtualAddress: 104 + SymbolName: main + Type: IMAGE_REL_AMD64_SECTION + - VirtualAddress: 139 + SymbolName: .text + Type: IMAGE_REL_AMD64_SECREL + - VirtualAddress: 143 + SymbolName: .text + Type: IMAGE_REL_AMD64_SECTION + - VirtualAddress: 174 + SymbolName: .text + Type: IMAGE_REL_AMD64_SECREL + - VirtualAddress: 178 + SymbolName: .text + Type: IMAGE_REL_AMD64_SECTION + - VirtualAddress: 196 + SymbolName: main + Type: IMAGE_REL_AMD64_SECREL + - VirtualAddress: 200 + SymbolName: main + Type: IMAGE_REL_AMD64_SECTION + - Name: '.debug$T' + Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ] + Alignment: 4 + SectionData: 040000000A000210700600000C0001000E0001120200000074000000001000000E0008107400000000000200011000001200011600000000021000006D61696E00F3F2F1 + Types: + - Kind: LF_POINTER + Pointer: + ReferentType: 1648 + Attrs: 65548 + - Kind: LF_ARGLIST + ArgList: + ArgIndices: [ 116, 4096 ] + - Kind: LF_PROCEDURE + Procedure: + ReturnType: 116 + CallConv: NearC + Options: [ None ] + ParameterCount: 2 + ArgumentList: 4097 + - Kind: LF_FUNC_ID + FuncId: + ParentScope: 0 + FunctionType: 4098 + Name: main + - Name: .pdata + Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_READ ] + Alignment: 4 + SectionData: 000000001C00000000000000 + Relocations: + - VirtualAddress: 0 + SymbolName: main + Type: IMAGE_REL_AMD64_ADDR32NB + - VirtualAddress: 4 + SymbolName: main + Type: IMAGE_REL_AMD64_ADDR32NB + - VirtualAddress: 8 + SymbolName: .xdata + Type: IMAGE_REL_AMD64_ADDR32NB +symbols: + - Name: .text + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 28 + NumberOfRelocations: 0 + NumberOfLinenumbers: 0 + CheckSum: 594448369 + Number: 1 + - Name: .data + Value: 0 + SectionNumber: 2 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 0 + NumberOfRelocations: 0 + NumberOfLinenumbers: 0 + CheckSum: 0 + Number: 2 + - Name: .bss + Value: 0 + SectionNumber: 3 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 0 + NumberOfRelocations: 0 + NumberOfLinenumbers: 0 + CheckSum: 0 + Number: 3 + - Name: .xdata + Value: 0 + SectionNumber: 4 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 8 + NumberOfRelocations: 0 + NumberOfLinenumbers: 0 + CheckSum: 1192424177 + Number: 4 + - Name: .drectve + Value: 0 + SectionNumber: 5 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 48 + NumberOfRelocations: 0 + NumberOfLinenumbers: 0 + CheckSum: 149686238 + Number: 5 + - Name: '.debug$S' + Value: 0 + SectionNumber: 6 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 324 + NumberOfRelocations: 8 + NumberOfLinenumbers: 0 + CheckSum: 4196717433 + Number: 6 + - Name: '.debug$T' + Value: 0 + SectionNumber: 7 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 68 + NumberOfRelocations: 0 + NumberOfLinenumbers: 0 + CheckSum: 485351071 + Number: 7 + - Name: .pdata + Value: 0 + SectionNumber: 8 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 12 + NumberOfRelocations: 3 + NumberOfLinenumbers: 0 + CheckSum: 722740324 + Number: 8 + - Name: main + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL +... Index: lld/trunk/test/COFF/Inputs/natvis-1.natvis =================================================================== --- lld/trunk/test/COFF/Inputs/natvis-1.natvis +++ lld/trunk/test/COFF/Inputs/natvis-1.natvis @@ -0,0 +1 @@ +1st Natvis Test Index: lld/trunk/test/COFF/Inputs/natvis-2.natvis =================================================================== --- lld/trunk/test/COFF/Inputs/natvis-2.natvis +++ lld/trunk/test/COFF/Inputs/natvis-2.natvis @@ -0,0 +1 @@ +Second Natvis Test Index: lld/trunk/test/COFF/Inputs/natvis-3.natvis =================================================================== --- lld/trunk/test/COFF/Inputs/natvis-3.natvis +++ lld/trunk/test/COFF/Inputs/natvis-3.natvis @@ -0,0 +1 @@ +Third Natvis Test Index: lld/trunk/test/COFF/pdb-natvis.test =================================================================== --- lld/trunk/test/COFF/pdb-natvis.test +++ lld/trunk/test/COFF/pdb-natvis.test @@ -0,0 +1,26 @@ +REQUIRES: diasdk + +RUN: yaml2obj %p/Inputs/generic.yaml > %t.obj +RUN: lld-link /DEBUG %t.obj /nodefaultlib /entry:main /NATVIS:%p/Inputs/natvis-1.natvis \ +RUN: /NATVIS:%p/Inputs/natvis-2.natvis /NATVIS:%p/Inputs/natvis-3.natvis /OUT:%t.exe \ +RUN: /PDB:%t.pdb +RUN: llvm-pdbutil pretty -injected-sources -injected-source-content %t.pdb | FileCheck \ +RUN: --check-prefix=CHECK-FIRST %s +RUN: llvm-pdbutil pretty -injected-sources -injected-source-content %t.pdb | FileCheck \ +RUN: --check-prefix=CHECK-SECOND %s +RUN: llvm-pdbutil pretty -injected-sources -injected-source-content %t.pdb | FileCheck \ +RUN: --check-prefix=CHECK-THIRD %s + +RUN: lld-link /DEBUG %t.obj /nodefaultlib /entry:main /NATVIS:%p/Inputs/test2.natvis \ +RUN: /OUT:%t.exe /PDB:%t.pdb 2>&1 | FileCheck --check-prefix=CHECK-MISSING %s + +CHECK-FIRST: {{.*}}natvis-1.natvis (16 bytes): obj=, vname={{.*}}natvis-1.natvis, crc=355285096, compression=None +CHECK-FIRST-NEXT: 1st Natvis Test + +CHECK-SECOND: {{.*}}natvis-2.natvis (19 bytes): obj=, vname={{.*}}natvis-2.natvis, crc=4252640062, compression=None +CHECK-SECOND-NEXT: Second Natvis Test + +CHECK-THIRD: {{.*}}natvis-3.natvis (18 bytes): obj=, vname={{.*}}natvis-3.natvis, crc=2069719849, compression=None +CHECK-THIRD-NEXT: Third Natvis Test + +CHECK-MISSING: Cannot open input file: {{.*}}test2.natvis \ No newline at end of file Index: lld/trunk/test/lit.cfg.py =================================================================== --- lld/trunk/test/lit.cfg.py +++ lld/trunk/test/lit.cfg.py @@ -85,6 +85,9 @@ if (config.llvm_libxml2_enabled == '1'): config.available_features.add('libxml2') +if config.have_dia_sdk: + config.available_features.add("diasdk") + tar_executable = lit.util.which('tar', config.environment['PATH']) if tar_executable: tar_version = subprocess.Popen( Index: lld/trunk/test/lit.site.cfg.py.in =================================================================== --- lld/trunk/test/lit.site.cfg.py.in +++ lld/trunk/test/lit.site.cfg.py.in @@ -1,5 +1,6 @@ @LIT_SITE_CFG_IN_HEADER@ +config.have_dia_sdk = "@LLVM_ENABLE_DIA_SDK@" config.llvm_src_root = "@LLVM_SOURCE_DIR@" config.llvm_obj_root = "@LLVM_BINARY_DIR@" config.llvm_tools_dir = "@LLVM_TOOLS_DIR@" Index: llvm/trunk/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h =================================================================== --- llvm/trunk/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h +++ llvm/trunk/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h @@ -10,6 +10,7 @@ #ifndef LLVM_DEBUGINFO_CODEVIEW_DEBUGSTRINGTABLESUBSECTION_H #define LLVM_DEBUGINFO_CODEVIEW_DEBUGSTRINGTABLESUBSECTION_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -66,19 +67,24 @@ uint32_t insert(StringRef S); // Return the ID for string S. Assumes S exists in the table. - uint32_t getStringId(StringRef S) const; + uint32_t getIdForString(StringRef S) const; + + StringRef getStringForId(uint32_t Id) const; uint32_t calculateSerializedSize() const override; Error commit(BinaryStreamWriter &Writer) const override; uint32_t size() const; - StringMap::const_iterator begin() const { return Strings.begin(); } + StringMap::const_iterator begin() const { + return StringToId.begin(); + } - StringMap::const_iterator end() const { return Strings.end(); } + StringMap::const_iterator end() const { return StringToId.end(); } private: - StringMap Strings; + DenseMap IdToString; + StringMap StringToId; uint32_t StringSize = 1; }; Index: llvm/trunk/include/llvm/DebugInfo/PDB/Native/HashTable.h =================================================================== --- llvm/trunk/include/llvm/DebugInfo/PDB/Native/HashTable.h +++ llvm/trunk/include/llvm/DebugInfo/PDB/Native/HashTable.h @@ -213,6 +213,7 @@ Deleted.clear(); } + bool empty() const { return size() == 0; } uint32_t capacity() const { return Buckets.size(); } uint32_t size() const { return Present.count(); } @@ -303,12 +304,12 @@ void grow() { uint32_t S = size(); + uint32_t MaxLoad = maxLoad(capacity()); if (S < maxLoad(capacity())) return; assert(capacity() != UINT32_MAX && "Can't grow Hash table!"); - uint32_t NewCapacity = - (capacity() <= INT32_MAX) ? capacity() * 2 : UINT32_MAX; + uint32_t NewCapacity = (capacity() <= INT32_MAX) ? MaxLoad * 2 : UINT32_MAX; // Growing requires rebuilding the table and re-hashing every item. Make a // copy with a larger capacity, insert everything into the copy, then swap Index: llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h =================================================================== --- llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -17,9 +17,11 @@ #include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" #include #include @@ -54,12 +56,34 @@ Error commit(StringRef Filename); Expected getNamedStreamIndex(StringRef Name) const; - Error addNamedStream(StringRef Name, uint32_t Size); + Error addNamedStream(StringRef Name, StringRef Data); + void addInjectedSource(StringRef Name, std::unique_ptr Buffer); private: + struct InjectedSourceDescriptor { + // The full name of the stream that contains the contents of this injected + // source. This is built as a concatenation of the literal "/src/files" + // plus the "vname". + std::string StreamName; + + // The exact name of the file name as specified by the user. + uint32_t NameIndex; + + // The string table index of the "vname" of the file. As far as we + // understand, this is the same as the name, except it is lowercased and + // forward slashes are converted to backslashes. + uint32_t VNameIndex; + std::unique_ptr Content; + }; + Expected finalizeMsfLayout(); + Expected allocateNamedStream(StringRef Name, uint32_t Size); void commitFpm(WritableBinaryStream &MsfBuffer, const msf::MSFLayout &Layout); + void commitInjectedSources(WritableBinaryStream &MsfBuffer, + const msf::MSFLayout &Layout); + void commitSrcHeaderBlock(WritableBinaryStream &MsfBuffer, + const msf::MSFLayout &Layout); BumpPtrAllocator &Allocator; @@ -71,7 +95,13 @@ std::unique_ptr Ipi; PDBStringTableBuilder Strings; + StringTableHashTraits InjectedSourceHashTraits; + HashTable InjectedSourceTable; + + SmallVector InjectedSources; + NamedStreamMap NamedStreams; + DenseMap NamedStreamData; }; } } Index: llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h =================================================================== --- llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h +++ llvm/trunk/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h @@ -31,6 +31,16 @@ namespace pdb { class PDBFileBuilder; +class PDBStringTableBuilder; + +struct StringTableHashTraits { + PDBStringTableBuilder *Table; + + explicit StringTableHashTraits(PDBStringTableBuilder &Table); + uint32_t hashLookupKey(StringRef S) const; + StringRef storageKeyToLookupKey(uint32_t Offset) const; + uint32_t lookupKeyToStorageKey(StringRef S); +}; class PDBStringTableBuilder { public: @@ -38,6 +48,9 @@ // Returns the ID for S. uint32_t insert(StringRef S); + uint32_t getIdForString(StringRef S) const; + StringRef getStringForId(uint32_t Id) const; + uint32_t calculateSerializedSize() const; Error commit(BinaryStreamWriter &Writer) const; Index: llvm/trunk/include/llvm/DebugInfo/PDB/Native/RawConstants.h =================================================================== --- llvm/trunk/include/llvm/DebugInfo/PDB/Native/RawConstants.h +++ llvm/trunk/include/llvm/DebugInfo/PDB/Native/RawConstants.h @@ -32,6 +32,8 @@ PdbImplVC140 = 20140508, }; +enum class PdbRaw_SrcHeaderBlockVer : uint32_t { SrcVerOne = 19980827 }; + enum class PdbRaw_FeatureSig : uint32_t { VC110 = PdbImplVC110, VC140 = PdbImplVC140, Index: llvm/trunk/include/llvm/DebugInfo/PDB/Native/RawTypes.h =================================================================== --- llvm/trunk/include/llvm/DebugInfo/PDB/Native/RawTypes.h +++ llvm/trunk/include/llvm/DebugInfo/PDB/Native/RawTypes.h @@ -328,6 +328,34 @@ const uint32_t PDBStringTableSignature = 0xEFFEEFFE; +/// The header preceding the /src/headerblock stream. +struct SrcHeaderBlockHeader { + support::ulittle32_t Version; // PdbRaw_SrcHeaderBlockVer enumeration. + support::ulittle32_t Size; // Size of entire stream. + uint64_t FileTime; // Time stamp (Windows FILETIME format). + support::ulittle32_t Age; // Age + uint8_t Padding[44]; // Pad to 64 bytes. +}; +static_assert(sizeof(SrcHeaderBlockHeader) == 64, "Incorrect struct size!"); + +/// A single file record entry within the /src/headerblock stream. +struct SrcHeaderBlockEntry { + support::ulittle32_t Size; // Record Length. + support::ulittle32_t Version; // PdbRaw_SrcHeaderBlockVer enumeration. + support::ulittle32_t CRC; // CRC of the original file contents. + support::ulittle32_t FileSize; // Size of original source file. + support::ulittle32_t FileNI; // String table index of file name. + support::ulittle32_t ObjNI; // String table index of object name. + support::ulittle32_t VFileNI; // String table index of virtual file name. + uint8_t Compression; // PDB_SourceCompression enumeration. + uint8_t IsVirtual; // Is this a virtual file (injected)? + short Padding; // Pad to 4 bytes. + char Reserved[8]; +}; + +constexpr int I = sizeof(SrcHeaderBlockEntry); +static_assert(sizeof(SrcHeaderBlockEntry) == 40, "Incorrect struct size!"); + } // namespace pdb } // namespace llvm Index: llvm/trunk/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp =================================================================== --- llvm/trunk/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp +++ llvm/trunk/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp @@ -109,7 +109,7 @@ } uint32_t DebugChecksumsSubsection::mapChecksumOffset(StringRef FileName) const { - uint32_t Offset = Strings.getStringId(FileName); + uint32_t Offset = Strings.getIdForString(FileName); auto Iter = OffsetMap.find(Offset); assert(Iter != OffsetMap.end()); return Iter->second; Index: llvm/trunk/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp =================================================================== --- llvm/trunk/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp +++ llvm/trunk/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp @@ -80,13 +80,13 @@ Ids.push_back(&M); std::sort(Ids.begin(), Ids.end(), [this](const T &L1, const T &L2) { - return Strings.getStringId(L1->getKey()) < - Strings.getStringId(L2->getKey()); + return Strings.getIdForString(L1->getKey()) < + Strings.getIdForString(L2->getKey()); }); for (const auto &Item : Ids) { CrossModuleImport Imp; - Imp.ModuleNameOffset = Strings.getStringId(Item->getKey()); + Imp.ModuleNameOffset = Strings.getIdForString(Item->getKey()); Imp.Count = Item->getValue().size(); if (auto EC = Writer.writeObject(Imp)) return EC; Index: llvm/trunk/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp =================================================================== --- llvm/trunk/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp +++ llvm/trunk/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp @@ -46,12 +46,15 @@ : DebugSubsection(DebugSubsectionKind::StringTable) {} uint32_t DebugStringTableSubsection::insert(StringRef S) { - auto P = Strings.insert({S, StringSize}); + auto P = StringToId.insert({S, StringSize}); // If a given string didn't exist in the string table, we want to increment - // the string table size. - if (P.second) + // the string table size and insert it into the reverse lookup. + if (P.second) { + IdToString.insert({P.first->getValue(), P.first->getKey()}); StringSize += S.size() + 1; // +1 for '\0' + } + return P.first->second; } @@ -67,7 +70,7 @@ if (auto EC = Writer.writeCString(StringRef())) return EC; - for (auto &Pair : Strings) { + for (auto &Pair : StringToId) { StringRef S = Pair.getKey(); uint32_t Offset = Begin + Pair.getValue(); Writer.setOffset(Offset); @@ -81,10 +84,16 @@ return Error::success(); } -uint32_t DebugStringTableSubsection::size() const { return Strings.size(); } +uint32_t DebugStringTableSubsection::size() const { return StringToId.size(); } + +uint32_t DebugStringTableSubsection::getIdForString(StringRef S) const { + auto Iter = StringToId.find(S); + assert(Iter != StringToId.end()); + return Iter->second; +} -uint32_t DebugStringTableSubsection::getStringId(StringRef S) const { - auto Iter = Strings.find(S); - assert(Iter != Strings.end()); +StringRef DebugStringTableSubsection::getStringForId(uint32_t Id) const { + auto Iter = IdToString.find(Id); + assert(Iter != IdToString.end()); return Iter->second; } Index: llvm/trunk/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp =================================================================== --- llvm/trunk/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp +++ llvm/trunk/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp @@ -73,5 +73,6 @@ if (auto EC = Writer.writeEnum(E)) return EC; } + assert(Writer.bytesRemaining() == 0); return Error::success(); } Index: llvm/trunk/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp =================================================================== --- llvm/trunk/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp +++ llvm/trunk/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp @@ -24,6 +24,8 @@ #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h" #include "llvm/Support/BinaryStream.h" #include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/Support/JamCRC.h" +#include "llvm/Support/Path.h" using namespace llvm; using namespace llvm::codeview; @@ -32,7 +34,8 @@ using namespace llvm::support; PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator) - : Allocator(Allocator) {} + : Allocator(Allocator), InjectedSourceHashTraits(Strings), + InjectedSourceTable(2, InjectedSourceHashTraits) {} PDBFileBuilder::~PDBFileBuilder() {} @@ -80,14 +83,45 @@ return *Gsi; } -Error PDBFileBuilder::addNamedStream(StringRef Name, uint32_t Size) { +Expected PDBFileBuilder::allocateNamedStream(StringRef Name, + uint32_t Size) { auto ExpectedStream = Msf->addStream(Size); - if (!ExpectedStream) - return ExpectedStream.takeError(); - NamedStreams.set(Name, *ExpectedStream); + if (ExpectedStream) + NamedStreams.set(Name, *ExpectedStream); + return ExpectedStream; +} + +Error PDBFileBuilder::addNamedStream(StringRef Name, StringRef Data) { + Expected ExpectedIndex = allocateNamedStream(Name, Data.size()); + if (!ExpectedIndex) + return ExpectedIndex.takeError(); + assert(NamedStreamData.count(*ExpectedIndex) == 0); + NamedStreamData[*ExpectedIndex] = Data; return Error::success(); } +void PDBFileBuilder::addInjectedSource(StringRef Name, + std::unique_ptr Buffer) { + // Stream names must be exact matches, since they get looked up in a hash + // table and the hash value is dependent on the exact contents of the string. + // link.exe lowercases a path and converts / to \, so we must do the same. + SmallString<64> VName; + sys::path::native(Name.lower(), VName); + + uint32_t NI = getStringTableBuilder().insert(Name); + uint32_t VNI = getStringTableBuilder().insert(VName); + + InjectedSourceDescriptor Desc; + Desc.Content = std::move(Buffer); + Desc.NameIndex = NI; + Desc.VNameIndex = VNI; + Desc.StreamName = "/src/files/"; + + Desc.StreamName += VName; + + InjectedSources.push_back(std::move(Desc)); +} + Expected PDBFileBuilder::finalizeMsfLayout() { if (Ipi && Ipi->getRecordCount() > 0) { @@ -101,15 +135,13 @@ uint32_t StringsLen = Strings.calculateSerializedSize(); - if (auto EC = addNamedStream("/names", StringsLen)) - return std::move(EC); - if (auto EC = addNamedStream("/LinkInfo", 0)) - return std::move(EC); + Expected SN = allocateNamedStream("/names", StringsLen); + if (!SN) + return SN.takeError(); + SN = allocateNamedStream("/LinkInfo", 0); + if (!SN) + return SN.takeError(); - if (Info) { - if (auto EC = Info->finalizeMsfLayout()) - return std::move(EC); - } if (Dbi) { if (auto EC = Dbi->finalizeMsfLayout()) return std::move(EC); @@ -132,6 +164,46 @@ } } + if (!InjectedSources.empty()) { + for (const auto &IS : InjectedSources) { + JamCRC CRC(0); + CRC.update(makeArrayRef(IS.Content->getBufferStart(), + IS.Content->getBufferSize())); + + SrcHeaderBlockEntry Entry; + ::memset(&Entry, 0, sizeof(SrcHeaderBlockEntry)); + Entry.Size = sizeof(SrcHeaderBlockEntry); + Entry.FileSize = IS.Content->getBufferSize(); + Entry.FileNI = IS.NameIndex; + Entry.VFileNI = IS.VNameIndex; + Entry.IsVirtual = 0; + Entry.Version = + static_cast(PdbRaw_SrcHeaderBlockVer::SrcVerOne); + Entry.CRC = CRC.getCRC(); + StringRef VName = getStringTableBuilder().getStringForId(IS.VNameIndex); + InjectedSourceTable.set_as(VName, std::move(Entry)); + } + + uint32_t SrcHeaderBlockSize = + sizeof(SrcHeaderBlockHeader) + + InjectedSourceTable.calculateSerializedLength(); + SN = allocateNamedStream("/src/headerblock", SrcHeaderBlockSize); + if (!SN) + return SN.takeError(); + for (const auto &IS : InjectedSources) { + SN = allocateNamedStream(IS.StreamName, IS.Content->getBufferSize()); + if (!SN) + return SN.takeError(); + } + } + + // Do this last, since it relies on the named stream map being complete, and + // that can be updated by previous steps in the finalization. + if (Info) { + if (auto EC = Info->finalizeMsfLayout()) + return std::move(EC); + } + return Msf->build(); } @@ -167,6 +239,45 @@ assert(FpmWriter.bytesRemaining() == 0); } +void PDBFileBuilder::commitSrcHeaderBlock(WritableBinaryStream &MsfBuffer, + const msf::MSFLayout &Layout) { + assert(!InjectedSourceTable.empty()); + + uint32_t SN = cantFail(getNamedStreamIndex("/src/headerblock")); + auto Stream = WritableMappedBlockStream::createIndexedStream( + Layout, MsfBuffer, SN, Allocator); + BinaryStreamWriter Writer(*Stream); + + SrcHeaderBlockHeader Header; + ::memset(&Header, 0, sizeof(Header)); + Header.Version = static_cast(PdbRaw_SrcHeaderBlockVer::SrcVerOne); + Header.Size = Writer.bytesRemaining(); + + cantFail(Writer.writeObject(Header)); + cantFail(InjectedSourceTable.commit(Writer)); + + assert(Writer.bytesRemaining() == 0); +} + +void PDBFileBuilder::commitInjectedSources(WritableBinaryStream &MsfBuffer, + const msf::MSFLayout &Layout) { + if (InjectedSourceTable.empty()) + return; + + commitSrcHeaderBlock(MsfBuffer, Layout); + + for (const auto &IS : InjectedSources) { + uint32_t SN = cantFail(getNamedStreamIndex(IS.StreamName)); + + auto SourceStream = WritableMappedBlockStream::createIndexedStream( + Layout, MsfBuffer, SN, Allocator); + BinaryStreamWriter SourceWriter(*SourceStream); + assert(SourceWriter.bytesRemaining() == IS.Content->getBufferSize()); + cantFail(SourceWriter.writeBytes( + arrayRefFromStringRef(IS.Content->getBuffer()))); + } +} + Error PDBFileBuilder::commit(StringRef Filename) { assert(!Filename.empty()); auto ExpectedLayout = finalizeMsfLayout(); @@ -219,6 +330,17 @@ if (auto EC = Strings.commit(NSWriter)) return EC; + for (const auto &NSE : NamedStreamData) { + if (NSE.second.empty()) + continue; + + auto NS = WritableMappedBlockStream::createIndexedStream( + Layout, Buffer, NSE.first, Allocator); + BinaryStreamWriter NSW(*NS); + if (auto EC = NSW.writeBytes(arrayRefFromStringRef(NSE.second))) + return EC; + } + if (Info) { if (auto EC = Info->commit(Layout, Buffer)) return EC; @@ -251,6 +373,8 @@ InfoStreamHeader *H = reinterpret_cast( FOB->getBufferStart() + InfoStreamFileOffset); + commitInjectedSources(Buffer, Layout); + // Set the build id at the very end, after every other byte of the PDB // has been written. // FIXME: Use a hash of the PDB rather than time(nullptr) for the signature. Index: llvm/trunk/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp =================================================================== --- llvm/trunk/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp +++ llvm/trunk/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp @@ -21,10 +21,33 @@ using namespace llvm::support::endian; using namespace llvm::pdb; +StringTableHashTraits::StringTableHashTraits(PDBStringTableBuilder &Table) + : Table(&Table) {} + +uint32_t StringTableHashTraits::hashLookupKey(StringRef S) const { + return Table->getIdForString(S); +} + +StringRef StringTableHashTraits::storageKeyToLookupKey(uint32_t Offset) const { + return Table->getStringForId(Offset); +} + +uint32_t StringTableHashTraits::lookupKeyToStorageKey(StringRef S) { + return Table->insert(S); +} + uint32_t PDBStringTableBuilder::insert(StringRef S) { return Strings.insert(S); } +uint32_t PDBStringTableBuilder::getIdForString(StringRef S) const { + return Strings.getIdForString(S); +} + +StringRef PDBStringTableBuilder::getStringForId(uint32_t Id) const { + return Strings.getStringForId(Id); +} + static uint32_t computeBucketCount(uint32_t NumStrings) { // The /names stream is basically an on-disk open-addressing hash table. // Hash collisions are resolved by linear probing. We cannot make Index: llvm/trunk/tools/llvm-pdbutil/DumpOutputStyle.h =================================================================== --- llvm/trunk/tools/llvm-pdbutil/DumpOutputStyle.h +++ llvm/trunk/tools/llvm-pdbutil/DumpOutputStyle.h @@ -74,6 +74,7 @@ Error dumpStreamSummary(); Error dumpSymbolStats(); Error dumpUdtStats(); + Error dumpNamedStreams(); Error dumpStringTable(); Error dumpStringTableFromPdb(); Error dumpStringTableFromObj(); Index: llvm/trunk/tools/llvm-pdbutil/DumpOutputStyle.cpp =================================================================== --- llvm/trunk/tools/llvm-pdbutil/DumpOutputStyle.cpp +++ llvm/trunk/tools/llvm-pdbutil/DumpOutputStyle.cpp @@ -90,6 +90,12 @@ P.NewLine(); } + if (opts::dump::DumpNamedStreams) { + if (auto EC = dumpNamedStreams()) + return EC; + P.NewLine(); + } + if (opts::dump::DumpStringTable) { if (auto EC = dumpStringTable()) return EC; @@ -909,6 +915,29 @@ return Error::success(); } +Error DumpOutputStyle::dumpNamedStreams() { + printHeader(P, "Named Streams"); + AutoIndent Indent(P, 2); + + if (File.isObj()) { + P.formatLine("Dumping Named Streams is only supported for PDB files."); + return Error::success(); + } + ExitOnError Err("Invalid PDB File: "); + + auto &IS = Err(File.pdb().getPDBInfoStream()); + const NamedStreamMap &NS = IS.getNamedStreams(); + for (const auto &Entry : NS.entries()) { + P.printLine(Entry.getKey()); + AutoIndent Indent2(P, 2); + P.formatLine("Index: {0}", Entry.getValue()); + P.formatLine("Size in bytes: {0}", + File.pdb().getStreamByteSize(Entry.getValue())); + } + + return Error::success(); +} + Error DumpOutputStyle::dumpStringTable() { printHeader(P, "String Table"); Index: llvm/trunk/tools/llvm-pdbutil/llvm-pdbutil.h =================================================================== --- llvm/trunk/tools/llvm-pdbutil/llvm-pdbutil.h +++ llvm/trunk/tools/llvm-pdbutil/llvm-pdbutil.h @@ -142,6 +142,7 @@ extern llvm::cl::opt DumpInlineeLines; extern llvm::cl::opt DumpXmi; extern llvm::cl::opt DumpXme; +extern llvm::cl::opt DumpNamedStreams; extern llvm::cl::opt DumpStringTable; extern llvm::cl::opt DumpTypes; extern llvm::cl::opt DumpTypeData; Index: llvm/trunk/tools/llvm-pdbutil/llvm-pdbutil.cpp =================================================================== --- llvm/trunk/tools/llvm-pdbutil/llvm-pdbutil.cpp +++ llvm/trunk/tools/llvm-pdbutil/llvm-pdbutil.cpp @@ -534,6 +534,10 @@ cl::cat(FileOptions), cl::sub(DumpSubcommand)); // MISCELLANEOUS OPTIONS +cl::opt DumpNamedStreams("named-streams", + cl::desc("dump PDB named stream table"), + cl::cat(MiscOptions), cl::sub(DumpSubcommand)); + cl::opt DumpStringTable("string-table", cl::desc("dump PDB String Table"), cl::cat(MiscOptions), cl::sub(DumpSubcommand));