Index: COFF/CMakeLists.txt =================================================================== --- COFF/CMakeLists.txt +++ COFF/CMakeLists.txt @@ -8,6 +8,7 @@ add_lld_library(lldCOFF Chunks.cpp + DebugTypes.cpp DLL.cpp Driver.cpp DriverUtils.cpp Index: COFF/DebugTypes.h =================================================================== --- COFF/DebugTypes.h +++ COFF/DebugTypes.h @@ -0,0 +1,61 @@ +//===- DebugTypes.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLD_COFF_DEBUGTYPES_H +#define LLD_COFF_DEBUGTYPES_H + +#include "llvm/Support/Error.h" + +namespace lld { +namespace coff { + +struct CVIndexMap; +struct MergedTypesState; +class ObjFile; + +class TpiSource { +public: + enum Kind { Regular, PCH, UsingPCH, PDB, UsingPDB }; + Kind kind() const { return SourceKind; } + + TpiSource(Kind K, ObjFile *F); + virtual ~TpiSource() {} + + /// Produce a mapping from the type and item indices used in the object + /// file to those in the destination PDB. + /// + /// If the object file uses a type server PDB (compiled with /Zi), merge TPI + /// and IPI from the type server PDB and return a map for it. Each unique type + /// server PDB is merged at most once, so this may return an existing index + /// mapping. + /// + /// If the object does not use a type server PDB (compiled with /Z7), we merge + /// all the type and item records from the .debug$S stream and fill in the + /// caller-provided ObjectIndexMap. + virtual llvm::Expected mergeDebugT(MergedTypesState *M, + CVIndexMap *IndexMap); + + /// Is this a dependent file that needs to be processed first, before other + /// OBJs? + virtual bool isDependency() const { return false; } + + static void forEachSource(std::function Fn); + + ObjFile *File; + +private: + const Kind SourceKind; +}; + +template +TpiSource *makeTpiSource(TpiSource::Kind K, T &&... Args); + +} // namespace coff +} // namespace lld + +#endif Index: COFF/DebugTypes.cpp =================================================================== --- COFF/DebugTypes.cpp +++ COFF/DebugTypes.cpp @@ -0,0 +1,433 @@ +//===- DebugTypes.cpp -----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DebugTypes.h" +#include "Chunks.h" +#include "Driver.h" +#include "InputFiles.h" +#include "MergedState.h" +#include "lld/Common/ErrorHandler.h" +#include "llvm/DebugInfo/CodeView/TypeHashing.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" +#include "llvm/DebugInfo/PDB/GenericError.h" +#include "llvm/DebugInfo/PDB/Native/InfoStream.h" +#include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/DebugInfo/PDB/Native/TpiStream.h" +#include "llvm/Object/COFF.h" +#include "llvm/Support/Path.h" + +using namespace lld; +using namespace lld::coff; +using namespace llvm; +using namespace llvm::codeview; + +/// Produce a mapping from the type and item indices used in the object +/// file to those in the destination PDB. +/// +/// If the object file uses a type server PDB (compiled with /Zi), merge TPI +/// and IPI from the type server PDB and return a map for it. Each unique type +/// server PDB is merged at most once, so this may return an existing index +/// mapping. +/// +/// If the object does not use a type server PDB (compiled with /Z7), we merge +/// all the type and item records from the .debug$S stream and fill in the +/// caller-provided ObjectIndexMap. + +namespace { +struct TypeServerSource : public TpiSource { + TypeServerSource(PDBInputFile *F) : TpiSource(PDB, nullptr), PDBFile(F) { + if (F->LoadErr && *F->LoadErr) + return; + pdb::PDBFile &File = F->Session->getPDBFile(); + auto ExpectedInfo = File.getPDBInfoStream(); + if (!ExpectedInfo) + return; + auto It = Mappings.emplace(ExpectedInfo->getGuid(), this); + assert(It.second); (It); + TSIndexMap.IsTypeServerMap = true; + } + + Expected mergeDebugT(MergedTypesState *M, + CVIndexMap *IndexMap) override; + bool isDependency() const override { return true; } + + PDBInputFile *PDBFile; + + CVIndexMap TSIndexMap; + + static std::map Mappings; +}; + +struct UseTypeServerSource : public TpiSource { + UseTypeServerSource(ObjFile *F, TypeServer2Record TS) + : TpiSource(UsingPDB, F), TypeServerDependency(TS) {} + + Expected mergeDebugT(MergedTypesState *M, + CVIndexMap *IndexMap) override; + + // Information about dependent PDB, that needs to be loaded in before merging + // this OBJ. Only valid when DebugTypesKind is UsingPDB + TypeServer2Record TypeServerDependency; +}; + +struct PrecompSource : public TpiSource { + PrecompSource(ObjFile *F) : TpiSource(PCH, F) { + if (!F->PCHSignature || !*F->PCHSignature) + fatal(toString(F) + " claims to be a precompiled headers object, but " + "does not have a valid signature!"); + auto It = Mappings.emplace(*F->PCHSignature, this); + if (!It.second) + fatal("A precompiled headers OBJ with the same signature has been " + "already provided! (" + + toString(It.first->second->File) + " and " + toString(File) + ")"); + PrecompIndexMap.IsPrecompiledTypeMap = true; + } + + Expected mergeDebugT(MergedTypesState *M, + CVIndexMap *IndexMap) override; + bool isDependency() const override { return true; } + + CVIndexMap PrecompIndexMap; + + static std::map Mappings; +}; + +struct UsePrecompSource : public TpiSource { + UsePrecompSource(ObjFile *F, PrecompRecord Precomp) + : TpiSource(UsingPCH, F), PrecompDependency(Precomp) {} + + Expected mergeDebugT(MergedTypesState *M, + CVIndexMap *IndexMap) override; + + // Information about dependent PCH, that needs to be loaded in before merging + // this OBJ. Only valid when DebugTypesKind is UsingPCH + PrecompRecord PrecompDependency; +}; +} // namespace + +static std::vector> GC; + +TpiSource::TpiSource(Kind K, ObjFile *F) : File(F), SourceKind(K) { + GC.push_back(std::unique_ptr(this)); +} + +template <> +coff::TpiSource *coff::makeTpiSource(coff::TpiSource::Kind K, + coff::ObjFile &File) { + switch (K) { + case TpiSource::Regular: + return new TpiSource(TpiSource::Regular, &File); + case TpiSource::PCH: + return new PrecompSource(&File); + default: + llvm_unreachable("Unhandled TpiSourceKind!"); + } +} +template <> +coff::TpiSource *coff::makeTpiSource(coff::TpiSource::Kind K, + coff::PDBInputFile &File) { + switch (K) { + case TpiSource::PDB: + return new TypeServerSource(&File); + default: + llvm_unreachable("Unhandled TpiSourceKind!"); + } +} +template <> +coff::TpiSource *coff::makeTpiSource(coff::TpiSource::Kind K, + coff::ObjFile &File, + llvm::codeview::TypeServer2Record &TS) { + switch (K) { + case TpiSource::UsingPDB: + return new UseTypeServerSource(&File, TS); + default: + llvm_unreachable("Unhandled TpiSourceKind!"); + } +} +template <> +coff::TpiSource *coff::makeTpiSource(coff::TpiSource::Kind K, + coff::ObjFile &File, + llvm::codeview::PrecompRecord &Precomp) { + switch (K) { + case TpiSource::UsingPCH: + return new UsePrecompSource(&File, Precomp); + default: + llvm_unreachable("Unhandled TpiSourceKind!"); + } +} + +void TpiSource::forEachSource(std::function Fn) { + for_each(GC, [&](std::unique_ptr &O) { Fn(O.get()); }); +} + +std::map TypeServerSource::Mappings; + +std::map PrecompSource::Mappings; + +// A COFF .debug$H section is currently a clang extension. This function checks +// if a .debug$H section is in a format that we expect / understand, so that we +// can ignore any sections which are coincidentally also named .debug$H but do +// not contain a format we recognize. +static bool canUseDebugH(ArrayRef DebugH) { + if (DebugH.size() < sizeof(object::debug_h_header)) + return false; + auto *Header = + reinterpret_cast(DebugH.data()); + DebugH = DebugH.drop_front(sizeof(object::debug_h_header)); + return Header->Magic == COFF::DEBUG_HASHES_SECTION_MAGIC && + Header->Version == 0 && + Header->HashAlgorithm == uint16_t(GlobalTypeHashAlg::SHA1_8) && + (DebugH.size() % 8 == 0); +} + +static Optional> getDebugH(ObjFile *File) { + SectionChunk *Sec = + SectionChunk::findByName(File->getDebugChunks(), ".debug$H"); + if (!Sec) + return llvm::None; + ArrayRef Contents = Sec->getContents(); + if (!canUseDebugH(Contents)) + return None; + return Contents; +} + +static ArrayRef +getHashesFromDebugH(ArrayRef DebugH) { + assert(canUseDebugH(DebugH)); + + DebugH = DebugH.drop_front(sizeof(object::debug_h_header)); + uint32_t Count = DebugH.size() / sizeof(GloballyHashedType); + return {reinterpret_cast(DebugH.data()), Count}; +} + +Expected TpiSource::mergeDebugT(MergedTypesState *M, + CVIndexMap *IndexMap) { + auto &Types = *File->DebugTypes; + if (Config->DebugGHashes) { + ArrayRef Hashes; + std::vector OwnedHashes; + if (Optional> DebugH = getDebugH(File)) + Hashes = getHashesFromDebugH(*DebugH); + else { + OwnedHashes = GloballyHashedType::hashTypes(Types); + Hashes = OwnedHashes; + } + + if (auto Err = mergeTypeAndIdRecords(M->GlobalIDTable, M->GlobalTypeTable, + IndexMap->TPIMap, Types, Hashes, + File->PCHSignature)) + fatal("codeview::mergeTypeAndIdRecords failed: " + + toString(std::move(Err))); + } else { + if (auto Err = + mergeTypeAndIdRecords(M->IDTable, M->TypeTable, IndexMap->TPIMap, + Types, File->PCHSignature)) + fatal("codeview::mergeTypeAndIdRecords failed: " + + toString(std::move(Err))); + } + return IndexMap; +} + +Expected TypeServerSource::mergeDebugT(MergedTypesState *M, + CVIndexMap *) { + auto ExpectedTpi = PDBFile->Session->getPDBFile().getPDBTpiStream(); + if (auto E = ExpectedTpi.takeError()) + fatal("Type server does not have TPI stream: " + toString(std::move(E))); + auto ExpectedIpi = PDBFile->Session->getPDBFile().getPDBIpiStream(); + if (auto E = ExpectedIpi.takeError()) + fatal("Type server does not have TPI stream: " + toString(std::move(E))); + + if (Config->DebugGHashes) { + // PDBs do not actually store global hashes, so when merging a type server + // PDB we have to synthesize global hashes. To do this, we first synthesize + // global hashes for the TPI stream, since it is independent, then we + // synthesize hashes for the IPI stream, using the hashes for the TPI stream + // as inputs. + auto TpiHashes = GloballyHashedType::hashTypes(ExpectedTpi->typeArray()); + auto IpiHashes = + GloballyHashedType::hashIds(ExpectedIpi->typeArray(), TpiHashes); + + Optional EndPrecomp; + // Merge TPI first, because the IPI stream will reference type indices. + if (auto Err = + mergeTypeRecords(M->GlobalTypeTable, TSIndexMap.TPIMap, + ExpectedTpi->typeArray(), TpiHashes, EndPrecomp)) + fatal("codeview::mergeTypeRecords failed: " + toString(std::move(Err))); + + // Merge IPI. + if (auto Err = mergeIdRecords(M->GlobalIDTable, TSIndexMap.TPIMap, + TSIndexMap.IPIMap, ExpectedIpi->typeArray(), + IpiHashes)) + fatal("codeview::mergeIdRecords failed: " + toString(std::move(Err))); + } else { + // Merge TPI first, because the IPI stream will reference type indices. + if (auto Err = mergeTypeRecords(M->TypeTable, TSIndexMap.TPIMap, + ExpectedTpi->typeArray())) + fatal("codeview::mergeTypeRecords failed: " + toString(std::move(Err))); + + // Merge IPI. + if (auto Err = mergeIdRecords(M->IDTable, TSIndexMap.TPIMap, + TSIndexMap.IPIMap, ExpectedIpi->typeArray())) + fatal("codeview::mergeIdRecords failed: " + toString(std::move(Err))); + } + return &TSIndexMap; +} + +Expected +UseTypeServerSource::mergeDebugT(MergedTypesState *M, CVIndexMap *IndexMap) { + + const codeview::GUID &TSId = TypeServerDependency.getGuid(); + StringRef TSPath = TypeServerDependency.getName(); + + TypeServerSource *PDBFile; + auto It = TypeServerSource::Mappings.find(TSId); + if (It != TypeServerSource::Mappings.end()) { + PDBFile = It->second; + } else { + // The file failed to load, lookup by name + PDBInputFile *PDB = PDBInputFile::findFromRecordPath(TSPath, File); + if (!PDB) + return createFileError(TSPath, errorCodeToError(std::error_code( + ENOENT, std::generic_category()))); + // If an error occurred during loading, throw it now + if (PDB->LoadErr && *PDB->LoadErr) + return createFileError(TSPath, std::move(*PDB->LoadErr)); + + PDBFile = (TypeServerSource *)PDB->DebugTypesObj; + } + + pdb::PDBFile &PDBSession = PDBFile->PDBFile->Session->getPDBFile(); + auto ExpectedInfo = PDBSession.getPDBInfoStream(); + if (!ExpectedInfo) + return &PDBFile->TSIndexMap; + + // Just because a file with a matching name was found and it was an actual + // PDB file doesn't mean it matches. For it to match the InfoStream's GUID + // must match the GUID specified in the TypeServer2 record. + if (ExpectedInfo->getGuid() != TypeServerDependency.getGuid() || + ExpectedInfo->getAge() != TypeServerDependency.getAge()) + return createFileError( + TSPath, + make_error(pdb::pdb_error_code::signature_out_of_date)); + + return &PDBFile->TSIndexMap; +} + +static bool equals_path(StringRef path1, StringRef path2) { +#if defined(_WIN32) + return path1.equals_lower(path2); +#else + return path1.equals(path2); +#endif +} + +// Find by name an OBJ provided on the command line +static PrecompSource *findObjByName(StringRef FileNameOnly) { + SmallString<128> CurrentPath; + + for (auto KV : PrecompSource::Mappings) { + StringRef CurrentFileName = sys::path::filename(KV.second->File->getName()); + + // Compare based solely on the file name (link.exe behavior) + if (equals_path(CurrentFileName, FileNameOnly)) + return KV.second; + }; + return nullptr; +} + +Expected findPrecompMap(ObjFile *File, + PrecompRecord &PR) { + // Cross-compile warning: given that Clang doesn't generate LF_PRECOMP + // records, we assume the OBJ comes from a Windows build of cl.exe. Thusly, + // the paths embedded in the OBJs are in the Windows format. + SmallString<128> PRFileName = sys::path::filename( + PR.getPrecompFilePath(), sys::path::Style::windows); + + PrecompSource *Precomp; + auto It = PrecompSource::Mappings.find(PR.getSignature()); + if (It != PrecompSource::Mappings.end()) { + Precomp = It->second; + } else { + // Lookup by name + Precomp = findObjByName(PRFileName); + } + + if (!Precomp) + return createFileError( + PRFileName, + make_error(pdb::pdb_error_code::external_cmdline_ref)); + + if (PR.getSignature() != File->PCHSignature) + return createFileError( + toString(File), + make_error(pdb::pdb_error_code::signature_out_of_date)); + + if (PR.getSignature() != *Precomp->File->PCHSignature) + return createFileError( + toString(Precomp->File), + make_error(pdb::pdb_error_code::signature_out_of_date)); + + return &Precomp->PrecompIndexMap; +} + +/// Merges a precompiled headers TPI map into the current TPI map. The +/// precompiled headers object will also be loaded and remapped in the +/// process. +static Expected +mergeInPrecompHeaderObj(ObjFile *File, CVIndexMap *IndexMap, + PrecompRecord &Precomp) { + auto E = findPrecompMap(File, Precomp); + if (!E) + return E.takeError(); + + const CVIndexMap *PrecompIndexMap = *E; + assert(PrecompIndexMap->IsPrecompiledTypeMap); + + if (PrecompIndexMap->TPIMap.empty()) + return PrecompIndexMap; + + assert(Precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex); + assert(Precomp.getTypesCount() <= PrecompIndexMap->TPIMap.size()); + // Use the previously remapped index map from the precompiled headers. + IndexMap->TPIMap.append(PrecompIndexMap->TPIMap.begin(), + PrecompIndexMap->TPIMap.begin() + + Precomp.getTypesCount()); + return IndexMap; +} + +Expected +UsePrecompSource::mergeDebugT(MergedTypesState *M, CVIndexMap *IndexMap) { + // This object was compiled with /Yu, so process the corresponding + // precompiled headers object (/Yc) first. Some type indices in the current + // object are referencing data in the precompiled headers object, so we need + // both to be loaded. + auto E = mergeInPrecompHeaderObj(File, IndexMap, PrecompDependency); + if (!E) + return E.takeError(); + + auto &Types = *File->DebugTypes; + + // Drop LF_PRECOMP record from the input stream, as it has been replaced + // with the precompiled headers Type stream in the mergeInPrecompHeaderObj() + // call above. Note that we can't just call Types.drop_front(), as we + // explicitly want to rebase the stream. + auto FirstType = Types.begin(); + Types.setUnderlyingStream( + Types.getUnderlyingStream().drop_front(FirstType->RecordData.size())); + + return TpiSource::mergeDebugT(M, IndexMap); +} + +Expected PrecompSource::mergeDebugT(MergedTypesState *M, + CVIndexMap *) { + // Note that we're not using the provided CVIndexMap. Instead, we use our + // local one. Precompiled headers objects need to save the index map for + // further reference by other objects which use the precompiled headers. + return TpiSource::mergeDebugT(M, &PrecompIndexMap); +} \ No newline at end of file Index: COFF/Driver.h =================================================================== --- COFF/Driver.h +++ COFF/Driver.h @@ -75,6 +75,8 @@ void enqueueArchiveMember(const Archive::Child &C, StringRef SymName, StringRef ParentName); + void enqueuePDB(StringRef Path) { enqueuePath(Path, false); } + MemoryBufferRef takeBuffer(std::unique_ptr MB); private: Index: COFF/Driver.cpp =================================================================== --- COFF/Driver.cpp +++ COFF/Driver.cpp @@ -170,6 +170,9 @@ case file_magic::coff_import_library: Symtab->addFile(make(MBRef)); break; + case file_magic::pdb: + Symtab->addFile(make(MBRef)); + break; case file_magic::coff_cl_gl_object: error(Filename + ": is not a native COFF file. Recompile without /GL"); break; Index: COFF/InputFiles.h =================================================================== --- COFF/InputFiles.h +++ COFF/InputFiles.h @@ -15,6 +15,8 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/Archive.h" #include "llvm/Object/COFF.h" @@ -51,11 +53,12 @@ class SectionChunk; class Symbol; class Undefined; +class TpiSource; // The root class of input files. class InputFile { public: - enum Kind { ArchiveKind, ObjectKind, ImportKind, BitcodeKind }; + enum Kind { ArchiveKind, ObjectKind, PDBKind, ImportKind, BitcodeKind }; Kind kind() const { return FileKind; } virtual ~InputFile() {} @@ -128,9 +131,6 @@ // Returns the underlying COFF file. COFFObjectFile *getCOFFObj() { return COFFObj.get(); } - // Whether the object was already merged into the final PDB or not - bool wasProcessedForPDB() const { return !!ModuleDBI; } - static std::vector Instances; // Flags in the absolute @feat.00 symbol if it is present. These usually @@ -161,6 +161,12 @@ // Tells whether this file was compiled with /hotpatch bool HotPatchable = false; + // If the OBJ has a .debug$T stream, this tells how it will be handled. + TpiSource *DebugTypesObj = nullptr; + + // The .debug$T stream if there's one. + llvm::Optional DebugTypes; + private: const coff_section* getSection(uint32_t I); const coff_section *getSection(COFFSymbolRef Sym) { @@ -170,6 +176,7 @@ void initializeChunks(); void initializeSymbols(); void initializeFlags(); + void initializeDependencies(); SectionChunk * readSection(uint32_t SectionNumber, @@ -243,6 +250,31 @@ std::vector Symbols; }; +// This is a PDB type server dependency, that is not a input file per se, but +// needs to be treated like one. Such files are discovered from the debug type +// stream. +class PDBInputFile : public InputFile { +public: + explicit PDBInputFile(MemoryBufferRef M) : InputFile(PDBKind, M) {} + static bool classof(const InputFile *F) { return F->kind() == PDBKind; } + void parse() override; + + static void enqueue(StringRef Path, ObjFile *FromFile); + + static PDBInputFile *findFromRecordPath(StringRef Path, ObjFile *FromFile); + + static std::map Instances; + + // Record possible errors while opening the PDB file + llvm::Optional LoadErr; + + // This is the actual interface to the PDB (if it was opened successfully) + std::unique_ptr Session; + + // If the PDB has a .debug$T stream, this tells how it will be handled. + TpiSource *DebugTypesObj; +}; + // This type represents import library members that contain DLL names // and symbols exported from the DLLs. See Microsoft PE/COFF spec. 7 // for details about the format. Index: COFF/InputFiles.cpp =================================================================== --- COFF/InputFiles.cpp +++ COFF/InputFiles.cpp @@ -9,6 +9,7 @@ #include "InputFiles.h" #include "Chunks.h" #include "Config.h" +#include "DebugTypes.h" #include "Driver.h" #include "SymbolTable.h" #include "Symbols.h" @@ -22,6 +23,7 @@ #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" +#include "llvm/DebugInfo/CodeView/TypeDeserializer.h" #include "llvm/Object/Binary.h" #include "llvm/Object/COFF.h" #include "llvm/Support/Casting.h" @@ -48,6 +50,7 @@ namespace coff { std::vector ObjFile::Instances; +std::map PDBInputFile::Instances; std::vector ImportFile::Instances; std::vector BitcodeFile::Instances; @@ -129,6 +132,7 @@ initializeChunks(); initializeSymbols(); initializeFlags(); + initializeDependencies(); } const coff_section* ObjFile::getSection(uint32_t I) { @@ -656,6 +660,119 @@ } } +static std::string getBaseName(ObjFile *File, StringRef TSPath) { + StringRef LocalPath = + !File->ParentName.empty() ? File->ParentName : File->getName(); + std::string Path = sys::path::parent_path(LocalPath); + // Currently, type server PDBs are only created by cl, which only runs + // on Windows, so we can assume type server paths are Windows style. + Path += sys::path::filename(TSPath, sys::path::Style::windows); + return Path; +} + +void ObjFile::initializeDependencies() { + if (!Config->Debug) + return; + + bool IsPCH = false; + + auto Data = getDebugSection(".debug$P"); + if (!Data.empty()) + IsPCH = true; + else + Data = getDebugSection(".debug$T"); + + if (Data.empty()) + return; + + CVTypeArray Types; + BinaryStreamReader Reader(Data, support::little); + cantFail(Reader.readArray(Types, Reader.getLength())); + + auto FirstType = Types.begin(); + if (FirstType == Types.end()) + return; + + DebugTypes.emplace(Types); + + if (IsPCH) { + DebugTypesObj = makeTpiSource(TpiSource::PCH, *this); + return; + } + + if (FirstType->kind() == LF_TYPESERVER2) { + auto TS = cantFail( + TypeDeserializer::deserializeAs(FirstType->data())); + DebugTypesObj = makeTpiSource(TpiSource::UsingPDB, *this, TS); + PDBInputFile::enqueue(TS.getName(), this); + return; + } + + if (FirstType->kind() == LF_PRECOMP) { + auto Precomp = cantFail( + TypeDeserializer::deserializeAs(FirstType->data())); + DebugTypesObj = makeTpiSource(TpiSource::UsingPCH, *this, Precomp); + return; + } + + DebugTypesObj = makeTpiSource(TpiSource::Regular, *this); +} + +static Optional findPath(std::string Path, ObjFile *FromFile) { + // Ensure the file exists before anything else. In some cases, if the path + // points to a removable device, enqueuePath() would fail with an error + // (EAGAIN, "resource unavailable try again") which we want to skip + // silently. + if (!llvm::sys::fs::exists(Path)) + Path = getBaseName(FromFile, Path); + if (llvm::sys::fs::exists(Path)) + return Path; + return None; +} + +PDBInputFile *PDBInputFile::findFromRecordPath(StringRef Path, + ObjFile *FromFile) { + auto P = findPath(Path, FromFile); + if (!P) + return nullptr; + auto It = PDBInputFile::Instances.find(*P); + if (It != PDBInputFile::Instances.end()) + return It->second; + return nullptr; +} + +void PDBInputFile::enqueue(StringRef Path, ObjFile *FromFile) { + auto P = findPath(Path, FromFile); + if (!P) + return; + auto It = PDBInputFile::Instances.emplace(*P, nullptr); + if (!It.second) + return; // already scheduled for load + Driver->enqueuePDB(*P); +} + +void PDBInputFile::parse() { + PDBInputFile::Instances[MB.getBufferIdentifier()] = this; + + std::unique_ptr ThisSession; + LoadErr.emplace(pdb::NativeSession::createFromPdb( + MemoryBuffer::getMemBuffer(MB, false), ThisSession)); + if (*LoadErr) + return; // fail silently at this point - the error will be handled later, + // when merging the debug type stream + + Session.reset(static_cast(ThisSession.release())); + + pdb::PDBFile &PDBFile = Session->getPDBFile(); + auto ExpectedInfo = PDBFile.getPDBInfoStream(); + // All PDB Files should have an Info stream. + if (!ExpectedInfo) { + LoadErr.emplace(ExpectedInfo.takeError()); + return; + } + DebugTypesObj = makeTpiSource(TpiSource::PDB, *this); +} + StringRef ltrim1(StringRef S, const char *Chars) { if (!S.empty() && strchr(Chars, S[0])) return S.substr(1); Index: COFF/MergedState.h =================================================================== --- COFF/MergedState.h +++ COFF/MergedState.h @@ -0,0 +1,63 @@ +//===- MergedState.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLD_COFF_MERGEDSTATE_H +#define LLD_COFF_MERGEDSTATE_H + +#include "llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h" +#include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h" +#include "llvm/Support/Allocator.h" + +namespace lld { +namespace coff { + +struct MergedTypesState { + MergedTypesState(llvm::BumpPtrAllocator &Alloc) + : TypeTable(Alloc), IDTable(Alloc), GlobalTypeTable(Alloc), + GlobalIDTable(Alloc) {} + + /// Get the type table or the global type table if /DEBUG:GHASH is enabled. + inline llvm::codeview::TypeCollection &getTypeTable() { + if (Config->DebugGHashes) + return GlobalTypeTable; + return TypeTable; + } + + /// Get the ID table or the global ID table if /DEBUG:GHASH is enabled. + inline llvm::codeview::TypeCollection &getIDTable() { + if (Config->DebugGHashes) + return GlobalIDTable; + return IDTable; + } + + /// Type records that will go into the PDB TPI stream. + llvm::codeview::MergingTypeTableBuilder TypeTable; + + /// Item records that will go into the PDB IPI stream. + llvm::codeview::MergingTypeTableBuilder IDTable; + + /// Type records that will go into the PDB TPI stream (for /DEBUG:GHASH) + llvm::codeview::GlobalTypeTableBuilder GlobalTypeTable; + + /// Item records that will go into the PDB IPI stream (for /DEBUG:GHASH) + llvm::codeview::GlobalTypeTableBuilder GlobalIDTable; +}; + +/// Map from type index and item index in a type server PDB to the +/// corresponding index in the destination PDB. +struct CVIndexMap { + llvm::SmallVector TPIMap; + llvm::SmallVector IPIMap; + bool IsTypeServerMap = false; + bool IsPrecompiledTypeMap = false; +}; + +} // namespace coff +} // namespace lld + +#endif Index: COFF/PDB.cpp =================================================================== --- COFF/PDB.cpp +++ COFF/PDB.cpp @@ -9,53 +9,33 @@ #include "PDB.h" #include "Chunks.h" #include "Config.h" +#include "DebugTypes.h" #include "Driver.h" +#include "MergedState.h" #include "SymbolTable.h" #include "Symbols.h" #include "Writer.h" #include "lld/Common/ErrorHandler.h" -#include "lld/Common/Timer.h" #include "lld/Common/Threads.h" +#include "lld/Common/Timer.h" +#include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h" #include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h" -#include "llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h" -#include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h" -#include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h" #include "llvm/DebugInfo/CodeView/RecordName.h" -#include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" +#include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/SymbolRecordHelpers.h" #include "llvm/DebugInfo/CodeView/SymbolSerializer.h" -#include "llvm/DebugInfo/CodeView/TypeDeserializer.h" -#include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h" #include "llvm/DebugInfo/CodeView/TypeIndexDiscovery.h" -#include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" #include "llvm/DebugInfo/MSF/MSFBuilder.h" -#include "llvm/DebugInfo/MSF/MSFCommon.h" -#include "llvm/DebugInfo/PDB/GenericError.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h" -#include "llvm/DebugInfo/PDB/Native/DbiStream.h" #include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h" #include "llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h" -#include "llvm/DebugInfo/PDB/Native/InfoStream.h" #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h" -#include "llvm/DebugInfo/PDB/Native/NativeSession.h" -#include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/DebugInfo/PDB/Native/PDBFileBuilder.h" -#include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h" #include "llvm/DebugInfo/PDB/Native/TpiHashing.h" -#include "llvm/DebugInfo/PDB/Native/TpiStream.h" #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h" -#include "llvm/DebugInfo/PDB/PDB.h" -#include "llvm/Object/COFF.h" -#include "llvm/Object/CVDebugRecord.h" -#include "llvm/Support/BinaryByteStream.h" -#include "llvm/Support/Endian.h" -#include "llvm/Support/Errc.h" -#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/JamCRC.h" #include "llvm/Support/Path.h" -#include "llvm/Support/ScopedPrinter.h" -#include using namespace lld; using namespace lld::coff; @@ -76,14 +56,6 @@ static Timer DiskCommitTimer("Commit to Disk", TotalPdbLinkTimer); namespace { -/// Map from type index and item index in a type server PDB to the -/// corresponding index in the destination PDB. -struct CVIndexMap { - SmallVector TPIMap; - SmallVector IPIMap; - bool IsTypeServerMap = false; - bool IsPrecompiledTypeMap = false; -}; class DebugSHandler; @@ -92,8 +64,7 @@ public: PDBLinker(SymbolTable *Symtab) - : Alloc(), Symtab(Symtab), Builder(Alloc), TypeTable(Alloc), - IDTable(Alloc), GlobalTypeTable(Alloc), GlobalIDTable(Alloc) { + : Alloc(), Symtab(Symtab), Builder(Alloc), M(Alloc) { // This isn't strictly necessary, but link.exe usually puts an empty string // as the first "valid" string in the string table, so we do the same in // order to maintain as much byte-for-byte compatibility as possible. @@ -109,51 +80,15 @@ /// Link CodeView from each object file in the symbol table into the PDB. void addObjectsToPDB(); + /// Link CodeView from a single object file into the target (output) PDB. /// When a precompiled headers object is linked, its TPI map might be provided /// externally. - void addObjFile(ObjFile *File, CVIndexMap *ExternIndexMap = nullptr); + void addDebug(TpiSource *Source); + + const CVIndexMap *mergeTypeRecords(TpiSource *Source, CVIndexMap *LocalMap); - /// Produce a mapping from the type and item indices used in the object - /// file to those in the destination PDB. - /// - /// If the object file uses a type server PDB (compiled with /Zi), merge TPI - /// and IPI from the type server PDB and return a map for it. Each unique type - /// server PDB is merged at most once, so this may return an existing index - /// mapping. - /// - /// If the object does not use a type server PDB (compiled with /Z7), we merge - /// all the type and item records from the .debug$S stream and fill in the - /// caller-provided ObjectIndexMap. - Expected mergeDebugT(ObjFile *File, - CVIndexMap *ObjectIndexMap); - - /// Reads and makes available a PDB. - Expected maybeMergeTypeServerPDB(ObjFile *File, - const CVType &FirstType); - - /// Merges a precompiled headers TPI map into the current TPI map. The - /// precompiled headers object will also be loaded and remapped in the - /// process. - Expected - mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType, - CVIndexMap *ObjectIndexMap); - - /// Reads and makes available a precompiled headers object. - /// - /// This is a requirement for objects compiled with cl.exe /Yu. In that - /// case, the referenced object (which was compiled with /Yc) has to be loaded - /// first. This is mainly because the current object's TPI stream has external - /// references to the precompiled headers object. - /// - /// If the precompiled headers object was already loaded, this function will - /// simply return its (remapped) TPI map. - Expected aquirePrecompObj(ObjFile *File, - PrecompRecord Precomp); - - /// Adds a precompiled headers object signature -> TPI mapping. - std::pair - registerPrecompiledHeaders(uint32_t Signature); + void addDebugSymbols(ObjFile *File, const CVIndexMap *IndexMap); void mergeSymbolRecords(ObjFile *File, const CVIndexMap &IndexMap, std::vector &StringTableRefs, @@ -163,20 +98,6 @@ void addSections(ArrayRef OutputSections, ArrayRef SectionTable); - /// Get the type table or the global type table if /DEBUG:GHASH is enabled. - TypeCollection &getTypeTable() { - if (Config->DebugGHashes) - return GlobalTypeTable; - return TypeTable; - } - - /// Get the ID table or the global ID table if /DEBUG:GHASH is enabled. - TypeCollection &getIDTable() { - if (Config->DebugGHashes) - return GlobalIDTable; - return IDTable; - } - /// Write the PDB to disk and store the Guid generated for it in *Guid. void commit(codeview::GUID *Guid); @@ -187,17 +108,7 @@ pdb::PDBFileBuilder Builder; - /// Type records that will go into the PDB TPI stream. - MergingTypeTableBuilder TypeTable; - - /// Item records that will go into the PDB IPI stream. - MergingTypeTableBuilder IDTable; - - /// Type records that will go into the PDB TPI stream (for /DEBUG:GHASH) - GlobalTypeTableBuilder GlobalTypeTable; - - /// Item records that will go into the PDB IPI stream (for /DEBUG:GHASH) - GlobalTypeTableBuilder GlobalIDTable; + MergedTypesState M; /// PDBs use a single global string table for filenames in the file checksum /// table. @@ -205,23 +116,7 @@ llvm::SmallString<128> NativePath; - /// A list of other PDBs which are loaded during the linking process and which - /// we need to keep around since the linking operation may reference pointers - /// inside of these PDBs. - llvm::SmallVector, 2> LoadedPDBs; - std::vector SectionMap; - - /// Type index mappings of type server PDBs that we've loaded so far. - std::map TypeServerIndexMappings; - - /// Type index mappings of precompiled objects type map that we've loaded so - /// far. - std::map PrecompTypeIndexMappings; - - /// List of TypeServer PDBs which cannot be loaded. - /// Cached to prevent repeated load attempts. - std::map MissingTypeServerPDBs; }; class DebugSHandler { @@ -231,7 +126,7 @@ ObjFile &File; /// The result of merging type indices. - const CVIndexMap &IndexMap; + const CVIndexMap *IndexMap; /// The DEBUG_S_STRINGTABLE subsection. These strings are referred to by /// index from other records in the .debug$S section. All of these strings @@ -261,7 +156,7 @@ std::vector StringTableReferences; public: - DebugSHandler(PDBLinker &Linker, ObjFile &File, const CVIndexMap &IndexMap) + DebugSHandler(PDBLinker &Linker, ObjFile &File, const CVIndexMap *IndexMap) : Linker(Linker), File(File), IndexMap(IndexMap) {} void handleDebugS(lld::coff::SectionChunk &DebugS); @@ -308,42 +203,6 @@ FileName = std::move(AbsoluteFileName); } -// A COFF .debug$H section is currently a clang extension. This function checks -// if a .debug$H section is in a format that we expect / understand, so that we -// can ignore any sections which are coincidentally also named .debug$H but do -// not contain a format we recognize. -static bool canUseDebugH(ArrayRef DebugH) { - if (DebugH.size() < sizeof(object::debug_h_header)) - return false; - auto *Header = - reinterpret_cast(DebugH.data()); - DebugH = DebugH.drop_front(sizeof(object::debug_h_header)); - return Header->Magic == COFF::DEBUG_HASHES_SECTION_MAGIC && - Header->Version == 0 && - Header->HashAlgorithm == uint16_t(GlobalTypeHashAlg::SHA1_8) && - (DebugH.size() % 8 == 0); -} - -static Optional> getDebugH(ObjFile *File) { - SectionChunk *Sec = - SectionChunk::findByName(File->getDebugChunks(), ".debug$H"); - if (!Sec) - return llvm::None; - ArrayRef Contents = Sec->getContents(); - if (!canUseDebugH(Contents)) - return None; - return Contents; -} - -static ArrayRef -getHashesFromDebugH(ArrayRef DebugH) { - assert(canUseDebugH(DebugH)); - - DebugH = DebugH.drop_front(sizeof(object::debug_h_header)); - uint32_t Count = DebugH.size() / sizeof(GloballyHashedType); - return {reinterpret_cast(DebugH.data()), Count}; -} - static void addTypeInfo(pdb::TpiStreamBuilder &TpiBuilder, TypeCollection &TypeTable) { // Start the TPI or IPI stream header. @@ -358,351 +217,6 @@ }); } -Expected -PDBLinker::mergeDebugT(ObjFile *File, CVIndexMap *ObjectIndexMap) { - ScopedTimer T(TypeMergingTimer); - - bool IsPrecompiledHeader = false; - - ArrayRef Data = File->getDebugSection(".debug$T"); - if (Data.empty()) { - // Try again, Microsoft precompiled headers use .debug$P instead of - // .debug$T - Data = File->getDebugSection(".debug$P"); - IsPrecompiledHeader = true; - } - if (Data.empty()) - return *ObjectIndexMap; // no debug info - - // Precompiled headers objects need to save the index map for further - // reference by other objects which use the precompiled headers. - if (IsPrecompiledHeader) { - uint32_t PCHSignature = File->PCHSignature.getValueOr(0); - if (PCHSignature == 0) - fatal("No signature found for the precompiled headers OBJ (" + - File->getName() + ")"); - - // When a precompiled headers object comes first on the command-line, we - // update the mapping here. Otherwise, if an object referencing the - // precompiled headers object comes first, the mapping is created in - // aquirePrecompObj(), thus we would skip this block. - if (!ObjectIndexMap->IsPrecompiledTypeMap) { - auto R = registerPrecompiledHeaders(PCHSignature); - if (R.second) - fatal( - "A precompiled headers OBJ with the same signature was already " - "provided! (" + - File->getName() + ")"); - - ObjectIndexMap = &R.first; - } - } - - BinaryByteStream Stream(Data, support::little); - CVTypeArray Types; - BinaryStreamReader Reader(Stream); - if (auto EC = Reader.readArray(Types, Reader.getLength())) - fatal("Reader::readArray failed: " + toString(std::move(EC))); - - auto FirstType = Types.begin(); - if (FirstType == Types.end()) - return *ObjectIndexMap; - - if (FirstType->kind() == LF_TYPESERVER2) { - // Look through type servers. If we've already seen this type server, - // don't merge any type information. - return maybeMergeTypeServerPDB(File, *FirstType); - } else if (FirstType->kind() == LF_PRECOMP) { - // This object was compiled with /Yu, so process the corresponding - // precompiled headers object (/Yc) first. Some type indices in the current - // object are referencing data in the precompiled headers object, so we need - // both to be loaded. - auto E = mergeInPrecompHeaderObj(File, *FirstType, ObjectIndexMap); - if (!E) - return E.takeError(); - - // Drop LF_PRECOMP record from the input stream, as it needs to be replaced - // with the precompiled headers object type stream. - // Note that we can't just call Types.drop_front(), as we explicitly want to - // rebase the stream. - Types.setUnderlyingStream( - Types.getUnderlyingStream().drop_front(FirstType->RecordData.size())); - } - - // Fill in the temporary, caller-provided ObjectIndexMap. - if (Config->DebugGHashes) { - ArrayRef Hashes; - std::vector OwnedHashes; - if (Optional> DebugH = getDebugH(File)) - Hashes = getHashesFromDebugH(*DebugH); - else { - OwnedHashes = GloballyHashedType::hashTypes(Types); - Hashes = OwnedHashes; - } - - if (auto Err = mergeTypeAndIdRecords(GlobalIDTable, GlobalTypeTable, - ObjectIndexMap->TPIMap, Types, Hashes, - File->PCHSignature)) - fatal("codeview::mergeTypeAndIdRecords failed: " + - toString(std::move(Err))); - } else { - if (auto Err = - mergeTypeAndIdRecords(IDTable, TypeTable, ObjectIndexMap->TPIMap, - Types, File->PCHSignature)) - fatal("codeview::mergeTypeAndIdRecords failed: " + - toString(std::move(Err))); - } - return *ObjectIndexMap; -} - -static Expected> -tryToLoadPDB(const codeview::GUID &GuidFromObj, StringRef TSPath) { - // Ensure the file exists before anything else. We want to return ENOENT, - // "file not found", even if the path points to a removable device (in which - // case the return message would be EAGAIN, "resource unavailable try again") - if (!llvm::sys::fs::exists(TSPath)) - return errorCodeToError(std::error_code(ENOENT, std::generic_category())); - - ErrorOr> MBOrErr = MemoryBuffer::getFile( - TSPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false); - if (!MBOrErr) - return errorCodeToError(MBOrErr.getError()); - - std::unique_ptr ThisSession; - if (auto EC = pdb::NativeSession::createFromPdb( - MemoryBuffer::getMemBuffer(Driver->takeBuffer(std::move(*MBOrErr)), - /*RequiresNullTerminator=*/false), - ThisSession)) - return std::move(EC); - - std::unique_ptr NS( - static_cast(ThisSession.release())); - pdb::PDBFile &File = NS->getPDBFile(); - auto ExpectedInfo = File.getPDBInfoStream(); - // All PDB Files should have an Info stream. - if (!ExpectedInfo) - return ExpectedInfo.takeError(); - - // Just because a file with a matching name was found and it was an actual - // PDB file doesn't mean it matches. For it to match the InfoStream's GUID - // must match the GUID specified in the TypeServer2 record. - if (ExpectedInfo->getGuid() != GuidFromObj) - return make_error(pdb::pdb_error_code::signature_out_of_date); - - return std::move(NS); -} - -Expected -PDBLinker::maybeMergeTypeServerPDB(ObjFile *File, const CVType &FirstType) { - TypeServer2Record TS; - if (auto EC = - TypeDeserializer::deserializeAs(const_cast(FirstType), TS)) - fatal("error reading record: " + toString(std::move(EC))); - - const codeview::GUID &TSId = TS.getGuid(); - StringRef TSPath = TS.getName(); - - // First, check if the PDB has previously failed to load. - auto PrevErr = MissingTypeServerPDBs.find(TSId); - if (PrevErr != MissingTypeServerPDBs.end()) - return createFileError( - TSPath, - make_error(PrevErr->second, inconvertibleErrorCode())); - - // Second, check if we already loaded a PDB with this GUID. Return the type - // index mapping if we have it. - auto Insertion = TypeServerIndexMappings.insert({TSId, CVIndexMap()}); - CVIndexMap &IndexMap = Insertion.first->second; - if (!Insertion.second) - return IndexMap; - - // Mark this map as a type server map. - IndexMap.IsTypeServerMap = true; - - // Check for a PDB at: - // 1. The given file path - // 2. Next to the object file or archive file - auto ExpectedSession = handleExpected( - tryToLoadPDB(TSId, TSPath), - [&]() { - StringRef LocalPath = - !File->ParentName.empty() ? File->ParentName : File->getName(); - SmallString<128> Path = sys::path::parent_path(LocalPath); - // Currently, type server PDBs are only created by cl, which only runs - // on Windows, so we can assume type server paths are Windows style. - sys::path::append( - Path, sys::path::filename(TSPath, sys::path::Style::windows)); - return tryToLoadPDB(TSId, Path); - }, - [&](std::unique_ptr EC) -> Error { - auto SysErr = EC->convertToErrorCode(); - // Only re-try loading if the previous error was "No such file or - // directory" - if (SysErr.category() == std::generic_category() && - SysErr.value() == ENOENT) - return Error::success(); - return Error(std::move(EC)); - }); - - if (auto E = ExpectedSession.takeError()) { - TypeServerIndexMappings.erase(TSId); - - // Flatten the error to a string, for later display, if the error occurs - // again on the same PDB. - std::string ErrMsg; - raw_string_ostream S(ErrMsg); - S << E; - MissingTypeServerPDBs.emplace(TSId, S.str()); - - return createFileError(TSPath, std::move(E)); - } - - pdb::NativeSession *Session = ExpectedSession->get(); - - // Keep a strong reference to this PDB, so that it's safe to hold pointers - // into the file. - LoadedPDBs.push_back(std::move(*ExpectedSession)); - - auto ExpectedTpi = Session->getPDBFile().getPDBTpiStream(); - if (auto E = ExpectedTpi.takeError()) - fatal("Type server does not have TPI stream: " + toString(std::move(E))); - auto ExpectedIpi = Session->getPDBFile().getPDBIpiStream(); - if (auto E = ExpectedIpi.takeError()) - fatal("Type server does not have TPI stream: " + toString(std::move(E))); - - if (Config->DebugGHashes) { - // PDBs do not actually store global hashes, so when merging a type server - // PDB we have to synthesize global hashes. To do this, we first synthesize - // global hashes for the TPI stream, since it is independent, then we - // synthesize hashes for the IPI stream, using the hashes for the TPI stream - // as inputs. - auto TpiHashes = GloballyHashedType::hashTypes(ExpectedTpi->typeArray()); - auto IpiHashes = - GloballyHashedType::hashIds(ExpectedIpi->typeArray(), TpiHashes); - - Optional EndPrecomp; - // Merge TPI first, because the IPI stream will reference type indices. - if (auto Err = mergeTypeRecords(GlobalTypeTable, IndexMap.TPIMap, - ExpectedTpi->typeArray(), TpiHashes, EndPrecomp)) - fatal("codeview::mergeTypeRecords failed: " + toString(std::move(Err))); - - // Merge IPI. - if (auto Err = - mergeIdRecords(GlobalIDTable, IndexMap.TPIMap, IndexMap.IPIMap, - ExpectedIpi->typeArray(), IpiHashes)) - fatal("codeview::mergeIdRecords failed: " + toString(std::move(Err))); - } else { - // Merge TPI first, because the IPI stream will reference type indices. - if (auto Err = mergeTypeRecords(TypeTable, IndexMap.TPIMap, - ExpectedTpi->typeArray())) - fatal("codeview::mergeTypeRecords failed: " + toString(std::move(Err))); - - // Merge IPI. - if (auto Err = mergeIdRecords(IDTable, IndexMap.TPIMap, IndexMap.IPIMap, - ExpectedIpi->typeArray())) - fatal("codeview::mergeIdRecords failed: " + toString(std::move(Err))); - } - - return IndexMap; -} - -Expected -PDBLinker::mergeInPrecompHeaderObj(ObjFile *File, const CVType &FirstType, - CVIndexMap *ObjectIndexMap) { - PrecompRecord Precomp; - if (auto EC = TypeDeserializer::deserializeAs(const_cast(FirstType), - Precomp)) - fatal("error reading record: " + toString(std::move(EC))); - - auto E = aquirePrecompObj(File, Precomp); - if (!E) - return E.takeError(); - - const CVIndexMap &PrecompIndexMap = *E; - assert(PrecompIndexMap.IsPrecompiledTypeMap); - - if (PrecompIndexMap.TPIMap.empty()) - return PrecompIndexMap; - - assert(Precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex); - assert(Precomp.getTypesCount() <= PrecompIndexMap.TPIMap.size()); - // Use the previously remapped index map from the precompiled headers. - ObjectIndexMap->TPIMap.append(PrecompIndexMap.TPIMap.begin(), - PrecompIndexMap.TPIMap.begin() + - Precomp.getTypesCount()); - return *ObjectIndexMap; -} - -static bool equals_path(StringRef path1, StringRef path2) { -#if defined(_WIN32) - return path1.equals_lower(path2); -#else - return path1.equals(path2); -#endif -} - -// Find by name an OBJ provided on the command line -static ObjFile *findObjByName(StringRef FileNameOnly) { - SmallString<128> CurrentPath; - - for (ObjFile *F : ObjFile::Instances) { - StringRef CurrentFileName = sys::path::filename(F->getName()); - - // Compare based solely on the file name (link.exe behavior) - if (equals_path(CurrentFileName, FileNameOnly)) - return F; - } - return nullptr; -} - -std::pair -PDBLinker::registerPrecompiledHeaders(uint32_t Signature) { - auto Insertion = PrecompTypeIndexMappings.insert({Signature, CVIndexMap()}); - CVIndexMap &IndexMap = Insertion.first->second; - if (!Insertion.second) - return {IndexMap, true}; - // Mark this map as a precompiled types map. - IndexMap.IsPrecompiledTypeMap = true; - return {IndexMap, false}; -} - -Expected -PDBLinker::aquirePrecompObj(ObjFile *File, PrecompRecord Precomp) { - // First, check if we already loaded the precompiled headers object with this - // signature. Return the type index mapping if we've already seen it. - auto R = registerPrecompiledHeaders(Precomp.getSignature()); - if (R.second) - return R.first; - - CVIndexMap &IndexMap = R.first; - - // Cross-compile warning: given that Clang doesn't generate LF_PRECOMP - // records, we assume the OBJ comes from a Windows build of cl.exe. Thusly, - // the paths embedded in the OBJs are in the Windows format. - SmallString<128> PrecompFileName = sys::path::filename( - Precomp.getPrecompFilePath(), sys::path::Style::windows); - - // link.exe requires that a precompiled headers object must always be provided - // on the command-line, even if that's not necessary. - auto PrecompFile = findObjByName(PrecompFileName); - if (!PrecompFile) - return createFileError( - PrecompFileName.str(), - make_error(pdb::pdb_error_code::external_cmdline_ref)); - - addObjFile(PrecompFile, &IndexMap); - - if (!PrecompFile->PCHSignature) - fatal(PrecompFile->getName() + " is not a precompiled headers object"); - - if (Precomp.getSignature() != PrecompFile->PCHSignature.getValueOr(0)) - return createFileError( - Precomp.getPrecompFilePath().str(), - make_error(pdb::pdb_error_code::signature_out_of_date)); - - return IndexMap; -} - static bool remapTypeIndex(TypeIndex &TI, ArrayRef TypeIndexMap) { if (TI.isSimple()) return true; @@ -1024,7 +538,7 @@ // An object file may have S_xxx_ID symbols, but these get converted to // "real" symbols in a PDB. - translateIdSymbols(RecordBytes, getIDTable()); + translateIdSymbols(RecordBytes, M.getIDTable()); Sym = CVSymbol(symbolKind(RecordBytes), RecordBytes); // If this record refers to an offset in the object file's string table, @@ -1153,7 +667,10 @@ break; } case DebugSubsectionKind::Symbols: { - Linker.mergeSymbolRecords(&File, IndexMap, StringTableReferences, + CVIndexMap TempIndexMap; + if (!IndexMap) + IndexMap = &TempIndexMap; + Linker.mergeSymbolRecords(&File, *IndexMap, StringTableReferences, SS.getRecordData()); break; } @@ -1212,18 +729,21 @@ File.ModuleDBI->addDebugSubsection(std::move(NewChecksums)); } -void PDBLinker::addObjFile(ObjFile *File, CVIndexMap *ExternIndexMap) { - if (File->wasProcessedForPDB()) - return; - // Add a module descriptor for every object file. We need to put an absolute - // path to the object into the PDB. If this is a plain object, we make its - // path absolute. If it's an object in an archive, we make the archive path - // absolute. +static StringRef getFileNamePath(InputFile *File, SmallString<128> &Path) { bool InArchive = !File->ParentName.empty(); - SmallString<128> Path = InArchive ? File->ParentName : File->getName(); + Path = InArchive ? File->ParentName : File->getName(); pdbMakeAbsolute(Path); StringRef Name = InArchive ? File->getName() : StringRef(Path); + return Name; +} +static void createModuleDBI(pdb::PDBFileBuilder &Builder, ObjFile *File) { + SmallString<128> Path; + StringRef Name = getFileNamePath(File, Path); + // Add a module descriptor for every object file. We need to put an absolute + // path to the object into the PDB. If this is a plain object, we make its + // path absolute. If it's an object in an archive, we make the archive path + // absolute. pdb::DbiStreamBuilder &DbiBuilder = Builder.getDbiBuilder(); File->ModuleDBI = &ExitOnErr(DbiBuilder.addModuleInfo(Name)); File->ModuleDBI->setObjFileName(Path); @@ -1238,31 +758,41 @@ File->ModuleDBI->setFirstSectionContrib(SC); break; } +} +static void warnUnusable(InputFile *F, Error E) { + if (!Config->WarnDebugInfoUnusable) { + consumeError(std::move(E)); + return; + } + auto Msg = "Cannot use debug info for '" + toString(F) + "' [LNK4099]"; + if (E) + warn(Msg + "\n>>> failed to load reference " + toString(std::move(E))); + else + warn(Msg); +} + +const CVIndexMap *PDBLinker::mergeTypeRecords(TpiSource *Source, CVIndexMap *LocalMap) { + ScopedTimer T(TypeMergingTimer); // Before we can process symbol substreams from .debug$S, we need to process // type information, file checksums, and the string table. Add type info to // the PDB first, so that we can get the map from object file type and item // indices to PDB type and item indices. - CVIndexMap ObjectIndexMap; - auto IndexMapResult = - mergeDebugT(File, ExternIndexMap ? ExternIndexMap : &ObjectIndexMap); + auto R = Source->mergeDebugT(&M, LocalMap); + T.stop(); // If the .debug$T sections fail to merge, assume there is no debug info. - if (!IndexMapResult) { - if (!Config->WarnDebugInfoUnusable) { - consumeError(IndexMapResult.takeError()); - return; - } - StringRef FileName = sys::path::filename(Path); - warn("Cannot use debug info for '" + FileName + "' [LNK4099]\n" + - ">>> failed to load reference " + - StringRef(toString(IndexMapResult.takeError()))); - return; + if (!R) { + warnUnusable(Source->File, R.takeError()); + return nullptr; } + return *R; +} - ScopedTimer T(SymbolMergingTimer); - - DebugSHandler DSH(*this, *File, *IndexMapResult); +void PDBLinker::addDebugSymbols(ObjFile *File, const CVIndexMap *IndexMap) { + ScopedTimer T2(SymbolMergingTimer); + pdb::DbiStreamBuilder &DbiBuilder = Builder.getDbiBuilder(); + DebugSHandler DSH(*this, *File, IndexMap); // Now do all live .debug$S and .debug$F sections. for (SectionChunk *DebugChunk : File->getDebugChunks()) { if (!DebugChunk->Live || DebugChunk->getSize() == 0) @@ -1294,6 +824,16 @@ DSH.finish(); } +void PDBLinker::addDebug(TpiSource *Source) { + CVIndexMap LocalMap; + const CVIndexMap *IndexMap = mergeTypeRecords(Source, &LocalMap); + + if (Source->kind() == TpiSource::PDB) + return; // No symbols in TypeServer PDBs + + addDebugSymbols(Source->File, IndexMap); +} + static PublicSym32 createPublic(Defined *Def) { PublicSym32 Pub(SymbolKind::S_PUB32); Pub.Name = Def->getName(); @@ -1315,16 +855,40 @@ // TpiData. void PDBLinker::addObjectsToPDB() { ScopedTimer T1(AddObjectsTimer); - for (ObjFile *File : ObjFile::Instances) - addObjFile(File); + + // Create module descriptors + for_each(ObjFile::Instances, + [&](ObjFile *Obj) { createModuleDBI(Builder, Obj); }); + + // Merge OBJs that do not have debug types + for_each(ObjFile::Instances, [&](ObjFile *Obj) { + if (Obj->DebugTypesObj) + return; + warnUnusable(Obj, Error::success()); + // Even if there're no types, still merge non-symbol .Debug$S and .Debug$F + // sections + addDebugSymbols(Obj, nullptr); + }); + + // Merge dependencies + TpiSource::forEachSource([&](TpiSource *Source) { + if (Source->isDependency()) + addDebug(Source); + }); + + // Merge regular and dependent OBJs + TpiSource::forEachSource([&](TpiSource *Source) { + if (!Source->isDependency()) + addDebug(Source); + }); Builder.getStringTableBuilder().setStrings(PDBStrTab); T1.stop(); // Construct TPI and IPI stream contents. ScopedTimer T2(TpiStreamLayoutTimer); - addTypeInfo(Builder.getTpiBuilder(), getTypeTable()); - addTypeInfo(Builder.getIpiBuilder(), getIDTable()); + addTypeInfo(Builder.getTpiBuilder(), M.getTypeTable()); + addTypeInfo(Builder.getIpiBuilder(), M.getIDTable()); T2.stop(); ScopedTimer T3(GlobalsLayoutTimer); Index: test/COFF/Inputs/pdb-type-server-valid-signature.yaml =================================================================== --- test/COFF/Inputs/pdb-type-server-valid-signature.yaml +++ test/COFF/Inputs/pdb-type-server-valid-signature.yaml @@ -69,7 +69,7 @@ - Kind: LF_TYPESERVER2 TypeServer2: Guid: '{8DABD2A0-28FF-CB43-9BAF-175B77B76414}' - Age: 18 + Age: 1 Name: 'pdb-diff-cl.pdb' - Name: '.text$mn' Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] Index: test/COFF/pdb-type-server-invalid-signature.yaml =================================================================== --- test/COFF/pdb-type-server-invalid-signature.yaml +++ test/COFF/pdb-type-server-invalid-signature.yaml @@ -4,7 +4,7 @@ # RUN: lld-link %t.obj -out:%t.exe -debug -pdb:%t.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s # RUN: cd %S -# CHECK: warning: Cannot use debug info for {{.*}}.obj +# CHECK: warning: Cannot use debug info for '{{.*}}.obj' # CHECK-NEXT: The signature does not match; the file(s) might be out of date # Also test a valid match @@ -14,7 +14,7 @@ # RUN: lld-link %t2.obj -out:%t2.exe -debug -pdb:%t2.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s -check-prefix=VALID-SIGNATURE -allow-empty # RUN: cd %S -# VALID-SIGNATURE-NOT: warning: Cannot use debug info for {{.*}}.obj +# VALID-SIGNATURE-NOT: warning: Cannot use debug info for '{{.*}}.obj' # VALID-SIGNATURE-NOT: The signature does not match; the file(s) might be out of date --- !COFF Index: test/COFF/precomp-link.test =================================================================== --- test/COFF/precomp-link.test +++ test/COFF/precomp-link.test @@ -6,12 +6,12 @@ RUN: lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-invalid.obj %S/Inputs/precomp.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE -RUN: not lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE-MISSING-PRECOMPOBJ +RUN: lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE-MISSING-PRECOMPOBJ -FAILURE: warning: Cannot use debug info for 'precomp-invalid.obj' [LNK4099] +FAILURE: warning: Cannot use debug info for '{{.*}}precomp-invalid.obj' [LNK4099] FAILURE-NEXT: failed to load reference '{{.*}}precomp.obj': The signature does not match; the file(s) might be out of date. -FAILURE-MISSING-PRECOMPOBJ: warning: Cannot use debug info for 'precomp-a.obj' [LNK4099] +FAILURE-MISSING-PRECOMPOBJ: warning: Cannot use debug info for '{{.*}}precomp-a.obj' [LNK4099] FAILURE-MISSING-PRECOMPOBJ-NEXT: failed to load reference '{{.*}}precomp.obj': The path to this file must be provided on the command-line CHECK: Types (TPI Stream)