Index: lld/test/COFF/pdb.test =================================================================== --- lld/test/COFF/pdb.test +++ lld/test/COFF/pdb.test @@ -6,6 +6,9 @@ # RUN: llvm-pdbdump pdb2yaml -stream-metadata -stream-directory -pdb-stream \ # RUN: -dbi-stream -ipi-stream -tpi-stream %t.pdb | FileCheck %s +# RUN: llvm-pdbdump raw -modules -section-map -section-headers -section-contribs \ +# RUN: -tpi-records %t.pdb | FileCheck -check-prefix RAW %s + # CHECK: MSF: # CHECK-NEXT: SuperBlock: # CHECK-NEXT: BlockSize: 4096 @@ -111,46 +114,9 @@ # CHECK-NEXT: BuildInfo: # CHECK-NEXT: ArgIndices: [ 4098, 4099, 4106, 4104, 4102 ] -# RAW: FileHeaders { -# RAW-NEXT: BlockSize: 4096 -# RAW-NEXT: FreeBlockMap: 1 -# RAW-NEXT: NumBlocks: 10 -# RAW-NEXT: NumDirectoryBytes: 48 -# RAW-NEXT: Unknown1: 0 -# RAW-NEXT: BlockMapAddr: 3 -# RAW-NEXT: NumDirectoryBlocks: 1 -# RAW-NEXT: DirectoryBlocks: [9] -# RAW-NEXT: NumStreams: 6 -# RAW-NEXT: } -# RAW-NEXT: Streams [ -# RAW-NEXT: Stream 0: [Old MSF Directory] (0 bytes) -# RAW-NEXT: Stream 1: [PDB Stream] (48 bytes) -# RAW-NEXT: Stream 2: [TPI Stream] (736 bytes) -# RAW-NEXT: Stream 3: [DBI Stream] (390 bytes) -# RAW-NEXT: Stream 4: [IPI Stream] (56 bytes) -# RAW-NEXT: Stream 5: [Section Header Data] (160 bytes) -# RAW-NEXT: ] -# RAW-NEXT: Msf Free Pages: [] -# RAW-NEXT: Orphaned Pages: [] -# RAW-NEXT: Multiply Used Pages: [] -# RAW-NEXT: Use After Free Pages: [] -# RAW-NEXT: StreamBlocks [ -# RAW-NEXT: Stream 0: [] -# RAW-NEXT: Stream 1: [5] -# RAW-NEXT: Stream 2: [7] -# RAW-NEXT: Stream 3: [6] -# RAW-NEXT: Stream 4: [8] -# RAW-NEXT: Stream 5: [4] -# RAW-NEXT: ] -# RAW-NEXT: PDB Stream { -# RAW-NEXT: Version: 20000404 -# RAW-NEXT: Signature: 0x0 -# RAW-NEXT: Age: 1 -# RAW-NEXT: Guid: {7EBCCC79-C488-0267-C898-06D7E94A8A10} -# RAW-NEXT: } -# RAW-NEXT: Type Info Stream (TPI) { +# RAW: Type Info Stream (TPI) { # RAW-NEXT: TPI Version: 20040203 -# RAW-NEXT: Record count: 17 +# RAW-NEXT: Record count: 5 # RAW-NEXT: Records [ # RAW-NEXT: { # RAW-NEXT: ArgList (0x1000) { @@ -205,126 +171,9 @@ # RAW-NEXT: ArgListType: () (0x1003) # RAW-NEXT: } # RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: FuncId (0x1005) { -# RAW-NEXT: TypeLeafKind: LF_FUNC_ID (0x1601) -# RAW-NEXT: ParentScope: 0x0 -# RAW-NEXT: FunctionType: int () (0x1004) -# RAW-NEXT: Name: main -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: FuncId (0x1006) { -# RAW-NEXT: TypeLeafKind: LF_FUNC_ID (0x1601) -# RAW-NEXT: ParentScope: 0x0 -# RAW-NEXT: FunctionType: int () (0x1001) -# RAW-NEXT: Name: foo -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x1007) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: 0x0 -# RAW-NEXT: StringData: D:\b -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x1008) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: 0x0 -# RAW-NEXT: StringData: C:\vs14\VC\BIN\amd64\cl.exe -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x1009) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: 0x0 -# RAW-NEXT: StringData: -Z7 -c -MT -IC:\vs14\VC\INCLUDE -IC:\vs14\VC\ATLMFC\INCLUDE -I"C:\Program Files (x86)\Windows Kits\10\include\10.0.10150.0\ucrt" -I"C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6\include\um" -I"C:\Program Files (x86)\Windows Kits\8.1\include\shared" -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringList (0x100A) { -# RAW-NEXT: TypeLeafKind: LF_SUBSTR_LIST (0x1604) -# RAW-NEXT: NumStrings: 1 -# RAW-NEXT: Strings [ -# RAW-NEXT: String: -Z7 -c -MT -IC:\vs14\VC\INCLUDE -IC:\vs14\VC\ATLMFC\INCLUDE -I"C:\Program Files (x86)\Windows Kits\10\include\10.0.10150.0\ucrt" -I"C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6\include\um" -I"C:\Program Files (x86)\Windows Kits\8.1\include\shared" (0x1009) -# RAW-NEXT: ] -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x100B) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: (-Z7 -c -MT -IC:\vs14\VC\INCLUDE -IC:\vs14\VC\ATLMFC\INCLUDE -I"C:\Program Files (x86)\Windows Kits\10\include\10.0.10150.0\ucrt" -I"C:\Program Files (x86)\Windows Kits\NETFXSDK\4.6\include\um" -I"C:\Program Files (x86)\Windows Kits\8.1\include\shared") (0x100A) -# RAW-NEXT: StringData: -I"C:\Program Files (x86)\Windows Kits\8.1\include\um" -I"C:\Program Files (x86)\Windows Kits\8.1\include\winrt" -TC -X -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x100C) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: 0x0 -# RAW-NEXT: StringData: ret42-main.c -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x100D) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: 0x0 -# RAW-NEXT: StringData: D:\b\vc140.pdb -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: BuildInfo (0x100E) { -# RAW-NEXT: TypeLeafKind: LF_BUILDINFO (0x1603) -# RAW-NEXT: NumArgs: 5 -# RAW-NEXT: Arguments [ -# RAW-NEXT: ArgType: D:\b (0x1007) -# RAW-NEXT: ArgType: C:\vs14\VC\BIN\amd64\cl.exe (0x1008) -# RAW-NEXT: ArgType: ret42-main.c (0x100C) -# RAW-NEXT: ArgType: D:\b\vc140.pdb (0x100D) -# RAW-NEXT: ArgType: -I"C:\Program Files (x86)\Windows Kits\8.1\include\um" -I"C:\Program Files (x86)\Windows Kits\8.1\include\winrt" -TC -X (0x100B) -# RAW-NEXT: ] -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: StringId (0x100F) { -# RAW-NEXT: TypeLeafKind: LF_STRING_ID (0x1605) -# RAW-NEXT: Id: 0x0 -# RAW-NEXT: StringData: ret42-sub.c -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: { -# RAW-NEXT: BuildInfo (0x1010) { -# RAW-NEXT: TypeLeafKind: LF_BUILDINFO (0x1603) -# RAW-NEXT: NumArgs: 5 -# RAW-NEXT: Arguments [ -# RAW-NEXT: ArgType: D:\b (0x1007) -# RAW-NEXT: ArgType: C:\vs14\VC\BIN\amd64\cl.exe (0x1008) -# RAW-NEXT: ArgType: ret42-sub.c (0x100F) -# RAW-NEXT: ArgType: D:\b\vc140.pdb (0x100D) -# RAW-NEXT: ArgType: -I"C:\Program Files (x86)\Windows Kits\8.1\include\um" -I"C:\Program Files (x86)\Windows Kits\8.1\include\winrt" -TC -X (0x100B) -# RAW-NEXT: ] -# RAW-NEXT: } -# RAW-NEXT: } -# RAW-NEXT: Hash { -# RAW-NEXT: Number of Hash Buckets: 4096 -# RAW-NEXT: Hash Key Size: 4 -# RAW-NEXT: Values: [] -# RAW-NEXT: Type Index Offsets: [] -# RAW-NEXT: Hash Adjustments: [] -# RAW-NEXT: } -# RAW-NEXT: ] -# RAW-NEXT: } -# RAW-NEXT: Type Info Stream (IPI) { -# RAW-NEXT: IPI Version: 20040203 -# RAW-NEXT: Record count: 0 -# RAW-NEXT: Records [ -# RAW-NEXT: Hash { -# RAW-NEXT: Number of Hash Buckets: 4096 -# RAW-NEXT: Hash Key Size: 4 -# RAW-NEXT: Values: [] -# RAW-NEXT: Type Index Offsets: [] -# RAW-NEXT: Hash Adjustments: [] -# RAW-NEXT: } +# RAW-NEXT: TypeIndexOffsets [ +# RAW-NEXT: Index: 0x1000, Offset: 0 +# RAW-NEXT: ] # RAW-NEXT: ] # RAW-NEXT: } # RAW-NEXT: DBI Stream { @@ -342,18 +191,16 @@ # RAW-NEXT: Modules [ # RAW-NEXT: { # RAW-NEXT: Name: * Linker * -# RAW-NEXT: Debug Stream Index: 65535 +# RAW-NEXT: Debug Stream Index: 9 # RAW-NEXT: Object File Name: # RAW-NEXT: Num Files: 0 # RAW-NEXT: Source File Name Idx: 0 # RAW-NEXT: Pdb File Name Idx: 0 # RAW-NEXT: Line Info Byte Size: 0 # RAW-NEXT: C13 Line Info Byte Size: 0 -# RAW-NEXT: Symbol Byte Size: 0 +# RAW-NEXT: Symbol Byte Size: 4 # RAW-NEXT: Type Server Index: 0 # RAW-NEXT: Has EC Info: No -# RAW-NEXT: 0 Contributing Source Files [ -# RAW-NEXT: ] # RAW-NEXT: } # RAW-NEXT: ] # RAW-NEXT: } @@ -481,7 +328,7 @@ # RAW-NEXT: SecName: 65535 # RAW-NEXT: ClassName: 65535 # RAW-NEXT: Offset: 0 -# RAW-NEXT: SecByteLength: 101 +# RAW-NEXT: SecByteLength: 122 # RAW-NEXT: } # RAW-NEXT: Entry { # RAW-NEXT: Flags [ (0x208) @@ -497,8 +344,6 @@ # RAW-NEXT: SecByteLength: 4294967295 # RAW-NEXT: } # RAW-NEXT: ] -# RAW-NEXT: Globals Stream not present -# RAW-NEXT: Publics Stream not present # RAW-NEXT: Section Headers [ # RAW-NEXT: { # RAW-NEXT: Name: .pdata @@ -548,7 +393,7 @@ # RAW-NEXT: } # RAW-NEXT: { # RAW-NEXT: Name: .rdata -# RAW-NEXT: Virtual Size: 101 +# RAW-NEXT: Virtual Size: 122 # RAW-NEXT: Virtual Address: 16384 # RAW-NEXT: Size of Raw Data: 512 # RAW-NEXT: File Pointer to Raw Data: 2560 @@ -562,5 +407,3 @@ # RAW-NEXT: ] # RAW-NEXT: } # RAW-NEXT: ] -# RAW-NEXT: New FPO [ -# RAW-NEXT: ] Index: llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h =================================================================== --- llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h +++ llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h @@ -13,6 +13,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" #include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/BinaryItemStream.h" @@ -63,6 +64,7 @@ private: uint32_t calculateHashBufferSize() const; + uint32_t calculateIndexOffsetSize() const; Error finalize(); msf::MSFBuilder &Msf; @@ -73,6 +75,7 @@ Optional VerHeader; std::vector> TypeRecords; std::vector TypeHashes; + std::vector TypeIndexOffsets; uint32_t HashStreamIndex = kInvalidStreamIndex; std::unique_ptr HashValueStream; Index: llvm/lib/DebugInfo/PDB/Native/TpiStream.cpp =================================================================== --- llvm/lib/DebugInfo/PDB/Native/TpiStream.cpp +++ llvm/lib/DebugInfo/PDB/Native/TpiStream.cpp @@ -97,7 +97,8 @@ uint32_t NumHashValues = Header->HashValueBuffer.Length / sizeof(ulittle32_t); - if (NumHashValues != NumTypeRecords()) + // FIXME: Allow zero hash tables for now. + if (NumHashValues != NumTypeRecords() && NumHashValues != 0) return make_error( raw_error_code::corrupt_file, "TPI hash count does not match with the number of type records."); @@ -124,8 +125,9 @@ // TPI hash table is a parallel array for the type records. // Verify that the hash values match with type records. - if (auto EC = verifyHashValues()) - return EC; + if (NumHashValues > 0) + if (auto EC = verifyHashValues()) + return EC; } return Error::success(); Index: llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp =================================================================== --- llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp +++ llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp @@ -45,7 +45,17 @@ void TpiStreamBuilder::addTypeRecord(ArrayRef Record, Optional Hash) { - TypeRecordBytes += Record.size(); + // If we just crossed an 8KB threshold, add a type index offset. + size_t NewSize = TypeRecordBytes + Record.size(); + constexpr size_t EightKB = 8 * 1024; + if (NewSize / EightKB > TypeRecordBytes / EightKB || TypeRecords.empty()) { + TypeIndexOffsets.push_back( + {codeview::TypeIndex(codeview::TypeIndex::FirstNonSimpleIndex + + TypeRecords.size()), + ulittle32_t(TypeRecordBytes)}); + } + TypeRecordBytes = NewSize; + TypeRecords.push_back(Record); if (Hash) TypeHashes.push_back(*Hash); @@ -58,7 +68,6 @@ TpiStreamHeader *H = Allocator.Allocate(); uint32_t Count = TypeRecords.size(); - uint32_t HashBufferSize = calculateHashBufferSize(); H->Version = *VerHeader; H->HeaderSize = sizeof(TpiStreamHeader); @@ -75,11 +84,15 @@ // the `HashStreamIndex` field of the `TpiStreamHeader`. Therefore, the data // begins at offset 0 of this independent stream. H->HashValueBuffer.Off = 0; - H->HashValueBuffer.Length = HashBufferSize; + H->HashValueBuffer.Length = calculateHashBufferSize(); + + // We never write any adjustments into our PDBs, so this is usually some + // offset with zero length. H->HashAdjBuffer.Off = H->HashValueBuffer.Off + H->HashValueBuffer.Length; H->HashAdjBuffer.Length = 0; + H->IndexOffsetBuffer.Off = H->HashAdjBuffer.Off + H->HashAdjBuffer.Length; - H->IndexOffsetBuffer.Length = 0; + H->IndexOffsetBuffer.Length = calculateIndexOffsetSize(); Header = H; return Error::success(); @@ -95,29 +108,37 @@ return TypeHashes.size() * sizeof(ulittle32_t); } +uint32_t TpiStreamBuilder::calculateIndexOffsetSize() const { + return TypeIndexOffsets.size() * sizeof(TypeIndexOffset); +} + Error TpiStreamBuilder::finalizeMsfLayout() { uint32_t Length = calculateSerializedLength(); if (auto EC = Msf.setStreamSize(Idx, Length)) return EC; - uint32_t HashBufferSize = calculateHashBufferSize(); + uint32_t HashStreamSize = + calculateHashBufferSize() + calculateIndexOffsetSize(); - if (HashBufferSize == 0) + if (HashStreamSize == 0) return Error::success(); - auto ExpectedIndex = Msf.addStream(HashBufferSize); + auto ExpectedIndex = Msf.addStream(HashStreamSize); if (!ExpectedIndex) return ExpectedIndex.takeError(); HashStreamIndex = *ExpectedIndex; - ulittle32_t *H = Allocator.Allocate(TypeHashes.size()); - MutableArrayRef HashBuffer(H, TypeHashes.size()); - for (uint32_t I = 0; I < TypeHashes.size(); ++I) { - HashBuffer[I] = TypeHashes[I] % MinTpiHashBuckets; + if (!TypeHashes.empty()) { + ulittle32_t *H = Allocator.Allocate(TypeHashes.size()); + MutableArrayRef HashBuffer(H, TypeHashes.size()); + for (uint32_t I = 0; I < TypeHashes.size(); ++I) { + HashBuffer[I] = TypeHashes[I] % MinTpiHashBuckets; + } + ArrayRef Bytes( + reinterpret_cast(HashBuffer.data()), + calculateHashBufferSize()); + HashValueStream = + llvm::make_unique(Bytes, llvm::support::little); } - ArrayRef Bytes(reinterpret_cast(HashBuffer.data()), - HashBufferSize); - HashValueStream = - llvm::make_unique(Bytes, llvm::support::little); return Error::success(); } @@ -141,8 +162,15 @@ auto HVS = WritableMappedBlockStream::createIndexedStream(Layout, Buffer, HashStreamIndex); BinaryStreamWriter HW(*HVS); - if (auto EC = HW.writeStreamRef(*HashValueStream)) - return EC; + if (HashValueStream) { + if (auto EC = HW.writeStreamRef(*HashValueStream)) + return EC; + } + + for (auto &IndexOffset : TypeIndexOffsets) { + if (auto EC = HW.writeObject(IndexOffset)) + return EC; + } } return Error::success(); Index: llvm/test/DebugInfo/PDB/pdb-yaml-types.test =================================================================== --- /dev/null +++ llvm/test/DebugInfo/PDB/pdb-yaml-types.test @@ -0,0 +1,74 @@ +; RUN: llvm-pdbdump pdb2yaml -tpi-stream %p/Inputs/big-read.pdb > %t.yaml +; RUN: FileCheck -check-prefix=YAML %s < %t.yaml +; RUN: llvm-pdbdump yaml2pdb %t.yaml -pdb %t.pdb +; RUN: llvm-pdbdump raw -tpi-records %t.pdb | FileCheck %s --check-prefix=PDB + +Only verify the beginning of the type stream. + +YAML: TpiStream: +YAML-NEXT: Version: VC80 +YAML-NEXT: Records: +YAML-NEXT: - Kind: LF_ARGLIST +YAML-NEXT: ArgList: +YAML-NEXT: ArgIndices: [ ] +YAML-NEXT: - Kind: LF_PROCEDURE +YAML-NEXT: Procedure: +YAML-NEXT: ReturnType: 3 +YAML-NEXT: CallConv: NearC +YAML-NEXT: Options: [ None ] +YAML-NEXT: ParameterCount: 0 +YAML-NEXT: ArgumentList: 4096 +YAML-NEXT: - Kind: LF_PROCEDURE +YAML-NEXT: Procedure: +YAML-NEXT: ReturnType: 116 +YAML-NEXT: CallConv: NearC +YAML-NEXT: Options: [ None ] +YAML-NEXT: ParameterCount: 0 +YAML-NEXT: ArgumentList: 4096 + +This test is mostly checking to make sure we include the type index offset +table, and eventually hash codes. The type index offsets should be similar to +what are already present in big-read.pdb. + +PDB: Type Info Stream (TPI) { +PDB-NEXT: TPI Version: 20040203 +PDB-NEXT: Record count: 728 +PDB-NEXT: Records [ +PDB-NEXT: { +PDB-NEXT: ArgList (0x1000) { +PDB-NEXT: TypeLeafKind: LF_ARGLIST (0x1201) +PDB-NEXT: NumArgs: 0 +PDB-NEXT: Arguments [ +PDB-NEXT: ] +PDB-NEXT: } +PDB-NEXT: } +PDB-NEXT: { +PDB-NEXT: Procedure (0x1001) { +PDB-NEXT: TypeLeafKind: LF_PROCEDURE (0x1008) +PDB-NEXT: ReturnType: void (0x3) +PDB-NEXT: CallingConvention: NearC (0x0) +PDB-NEXT: FunctionOptions [ (0x0) +PDB-NEXT: ] +PDB-NEXT: NumParameters: 0 +PDB-NEXT: ArgListType: () (0x1000) +PDB-NEXT: } +PDB-NEXT: } +PDB-NEXT: { +PDB-NEXT: Procedure (0x1002) { +PDB-NEXT: TypeLeafKind: LF_PROCEDURE (0x1008) +PDB-NEXT: ReturnType: int (0x74) +PDB-NEXT: CallingConvention: NearC (0x0) +PDB-NEXT: FunctionOptions [ (0x0) +PDB-NEXT: ] +PDB-NEXT: NumParameters: 0 +PDB-NEXT: ArgListType: () (0x1000) +PDB-NEXT: } +PDB-NEXT: } +... +PDB: TypeIndexOffsets [ +PDB-NEXT: Index: 0x1000, Offset: 0 +PDB-NEXT: Index: 0x106c, Offset: 8,116 +PDB-NEXT: Index: 0x1118, Offset: 16,372 +PDB-NEXT: Index: 0x11df, Offset: 24,564 +PDB-NEXT: Index: 0x128e, Offset: 32,752 +PDB-NEXT: ]