diff --git a/lld/COFF/DebugTypes.h b/lld/COFF/DebugTypes.h --- a/lld/COFF/DebugTypes.h +++ b/lld/COFF/DebugTypes.h @@ -26,35 +26,55 @@ namespace coff { class ObjFile; +class PDBInputFile; +struct CVIndexMap; +class TypeMerger; class TpiSource { public: enum TpiKind { Regular, PCH, UsingPCH, PDB, UsingPDB }; - TpiSource(TpiKind k, const ObjFile *f); - virtual ~TpiSource() {} + TpiSource(TpiKind k, ObjFile *f); + virtual ~TpiSource(); - const TpiKind kind; - const ObjFile *file; -}; + /// Produce a mapping from the type and item indices used in the object + /// file to those in the destination PDB. + /// + /// If the object file uses a type server PDB (compiled with /Zi), merge TPI + /// and IPI from the type server PDB and return a map for it. Each unique type + /// server PDB is merged at most once, so this may return an existing index + /// mapping. + /// + /// If the object does not use a type server PDB (compiled with /Z7), we merge + /// all the type and item records from the .debug$S stream and fill in the + /// caller-provided ObjectIndexMap. + virtual llvm::Expected mergeDebugT(TypeMerger *m, + CVIndexMap *indexMap); + /// Is this a dependent file that needs to be processed first, before other + /// OBJs? + virtual bool isDependency() const { return false; } + + static void forEachSource(llvm::function_ref fn); -TpiSource *makeTpiSource(const ObjFile *f); -TpiSource *makeUseTypeServerSource(const ObjFile *f, - const llvm::codeview::TypeServer2Record *ts); -TpiSource *makePrecompSource(const ObjFile *f); -TpiSource *makeUsePrecompSource(const ObjFile *f, - const llvm::codeview::PrecompRecord *precomp); + static uint32_t countTypeServerPDBs(); + static uint32_t countPrecompObjs(); -void loadTypeServerSource(llvm::MemoryBufferRef m); + /// Clear global data structures for TpiSources. + static void clear(); -// Temporary interface to get the dependency -template const T &retrieveDependencyInfo(const TpiSource *source); + const TpiKind kind; + ObjFile *file; +}; -// Temporary interface until we move PDBLinker::maybeMergeTypeServerPDB here -llvm::Expected -findTypeServerSource(const ObjFile *f); +TpiSource *makeTpiSource(ObjFile *file); +TpiSource *makeTypeServerSource(PDBInputFile *pdbInputFile); +TpiSource *makeUseTypeServerSource(ObjFile *file, + llvm::codeview::TypeServer2Record ts); +TpiSource *makePrecompSource(ObjFile *file); +TpiSource *makeUsePrecompSource(ObjFile *file, + llvm::codeview::PrecompRecord ts); } // namespace coff } // namespace lld -#endif \ No newline at end of file +#endif diff --git a/lld/COFF/DebugTypes.cpp b/lld/COFF/DebugTypes.cpp --- a/lld/COFF/DebugTypes.cpp +++ b/lld/COFF/DebugTypes.cpp @@ -7,15 +7,20 @@ //===----------------------------------------------------------------------===// #include "DebugTypes.h" +#include "Chunks.h" #include "Driver.h" #include "InputFiles.h" +#include "TypeMerger.h" #include "lld/Common/ErrorHandler.h" #include "lld/Common/Memory.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeRecordHelpers.h" +#include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" #include "llvm/DebugInfo/PDB/GenericError.h" #include "llvm/DebugInfo/PDB/Native/InfoStream.h" #include "llvm/DebugInfo/PDB/Native/NativeSession.h" #include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/DebugInfo/PDB/Native/TpiStream.h" #include "llvm/Support/Path.h" using namespace llvm; @@ -33,36 +38,40 @@ // before any dependent OBJ. class TypeServerSource : public TpiSource { public: - explicit TypeServerSource(MemoryBufferRef m, llvm::pdb::NativeSession *s) - : TpiSource(PDB, nullptr), session(s), mb(m) {} - - // Queue a PDB type server for loading in the COFF Driver - static void enqueue(const ObjFile *dependentFile, - const TypeServer2Record &ts); - - // Create an instance - static Expected getInstance(MemoryBufferRef m); - - // Fetch the PDB instance loaded for a corresponding dependent OBJ. - static Expected - findFromFile(const ObjFile *dependentFile); - - static std::map> - instances; - - // The interface to the PDB (if it was opened successfully) - std::unique_ptr session; - -private: - MemoryBufferRef mb; + explicit TypeServerSource(PDBInputFile *f) + : TpiSource(PDB, nullptr), pdbInputFile(f) { + if (f->loadErr && *f->loadErr) + return; + pdb::PDBFile &file = f->session->getPDBFile(); + auto expectedInfo = file.getPDBInfoStream(); + if (!expectedInfo) + return; + auto it = mappings.emplace(expectedInfo->getGuid(), this); + assert(it.second); + (void)it; + tsIndexMap.isTypeServerMap = true; + } + + Expected mergeDebugT(TypeMerger *m, + CVIndexMap *indexMap) override; + bool isDependency() const override { return true; } + + PDBInputFile *pdbInputFile = nullptr; + + CVIndexMap tsIndexMap; + + static std::map mappings; }; // This class represents the debug type stream of an OBJ file that depends on a // PDB type server (see TypeServerSource). class UseTypeServerSource : public TpiSource { public: - UseTypeServerSource(const ObjFile *f, const TypeServer2Record *ts) - : TpiSource(UsingPDB, f), typeServerDependency(*ts) {} + UseTypeServerSource(ObjFile *f, TypeServer2Record ts) + : TpiSource(UsingPDB, f), typeServerDependency(ts) {} + + Expected mergeDebugT(TypeMerger *m, + CVIndexMap *indexMap) override; // Information about the PDB type server dependency, that needs to be loaded // in before merging this OBJ. @@ -75,15 +84,35 @@ // such files, clang does not. class PrecompSource : public TpiSource { public: - PrecompSource(const ObjFile *f) : TpiSource(PCH, f) {} + PrecompSource(ObjFile *f) : TpiSource(PCH, f) { + if (!f->pchSignature || !*f->pchSignature) + fatal(toString(f) + + " claims to be a PCH object, but does not have a valid signature"); + auto it = mappings.emplace(*f->pchSignature, this); + if (!it.second) + fatal("a PCH object with the same signature has already been provided (" + + toString(it.first->second->file) + " and " + toString(file) + ")"); + precompIndexMap.isPrecompiledTypeMap = true; + } + + Expected mergeDebugT(TypeMerger *m, + CVIndexMap *indexMap) override; + bool isDependency() const override { return true; } + + CVIndexMap precompIndexMap; + + static std::map mappings; }; // This class represents the debug type stream of an OBJ file that depends on a // Microsoft precompiled headers OBJ (see PrecompSource). class UsePrecompSource : public TpiSource { public: - UsePrecompSource(const ObjFile *f, const PrecompRecord *precomp) - : TpiSource(UsingPCH, f), precompDependency(*precomp) {} + UsePrecompSource(ObjFile *f, PrecompRecord precomp) + : TpiSource(UsingPCH, f), precompDependency(precomp) {} + + Expected mergeDebugT(TypeMerger *m, + CVIndexMap *indexMap) override; // Information about the Precomp OBJ dependency, that needs to be loaded in // before merging this OBJ. @@ -91,175 +120,363 @@ }; } // namespace -TpiSource::TpiSource(TpiKind k, const ObjFile *f) : kind(k), file(f) {} +static std::vector gc; -TpiSource *lld::coff::makeTpiSource(const ObjFile *f) { - return make(TpiSource::Regular, f); +TpiSource::TpiSource(TpiKind k, ObjFile *f) : kind(k), file(f) { + gc.push_back(this); } -TpiSource *lld::coff::makeUseTypeServerSource(const ObjFile *f, - const TypeServer2Record *ts) { - TypeServerSource::enqueue(f, *ts); - return make(f, ts); +// Vtable key method. +TpiSource::~TpiSource() = default; + +TpiSource *lld::coff::makeTpiSource(ObjFile *file) { + return make(TpiSource::Regular, file); } -TpiSource *lld::coff::makePrecompSource(const ObjFile *f) { - return make(f); +TpiSource *lld::coff::makeTypeServerSource(PDBInputFile *pdbInputFile) { + return make(pdbInputFile); } -TpiSource *lld::coff::makeUsePrecompSource(const ObjFile *f, - const PrecompRecord *precomp) { - return make(f, precomp); +TpiSource *lld::coff::makeUseTypeServerSource(ObjFile *file, + TypeServer2Record ts) { + return make(file, ts); } -namespace lld { -namespace coff { -template <> -const PrecompRecord &retrieveDependencyInfo(const TpiSource *source) { - assert(source->kind == TpiSource::UsingPCH); - return ((const UsePrecompSource *)source)->precompDependency; +TpiSource *lld::coff::makePrecompSource(ObjFile *file) { + return make(file); } -template <> -const TypeServer2Record &retrieveDependencyInfo(const TpiSource *source) { - assert(source->kind == TpiSource::UsingPDB); - return ((const UseTypeServerSource *)source)->typeServerDependency; +TpiSource *lld::coff::makeUsePrecompSource(ObjFile *file, + PrecompRecord precomp) { + return make(file, precomp); } -} // namespace coff -} // namespace lld - -std::map> - TypeServerSource::instances; - -// Make a PDB path assuming the PDB is in the same folder as the OBJ -static std::string getPdbBaseName(const ObjFile *file, StringRef tSPath) { - StringRef localPath = - !file->parentName.empty() ? file->parentName : file->getName(); - SmallString<128> path = sys::path::parent_path(localPath); - - // Currently, type server PDBs are only created by MSVC cl, which only runs - // on Windows, so we can assume type server paths are Windows style. - sys::path::append(path, sys::path::filename(tSPath, sys::path::Style::windows)); - return std::string(path.str()); + +void TpiSource::forEachSource(llvm::function_ref fn) { + for_each(gc, fn); } -// The casing of the PDB path stamped in the OBJ can differ from the actual path -// on disk. With this, we ensure to always use lowercase as a key for the -// PDBInputFile::Instances map, at least on Windows. -static std::string normalizePdbPath(StringRef path) { -#if defined(_WIN32) - return path.lower(); -#else // LINUX - return std::string(path); -#endif +std::map TypeServerSource::mappings; + +std::map PrecompSource::mappings; + +// A COFF .debug$H section is currently a clang extension. This function checks +// if a .debug$H section is in a format that we expect / understand, so that we +// can ignore any sections which are coincidentally also named .debug$H but do +// not contain a format we recognize. +static bool canUseDebugH(ArrayRef debugH) { + if (debugH.size() < sizeof(object::debug_h_header)) + return false; + auto *header = + reinterpret_cast(debugH.data()); + debugH = debugH.drop_front(sizeof(object::debug_h_header)); + return header->Magic == COFF::DEBUG_HASHES_SECTION_MAGIC && + header->Version == 0 && + header->HashAlgorithm == uint16_t(GlobalTypeHashAlg::SHA1_8) && + (debugH.size() % 8 == 0); +} + +static Optional> getDebugH(ObjFile *file) { + SectionChunk *sec = + SectionChunk::findByName(file->getDebugChunks(), ".debug$H"); + if (!sec) + return llvm::None; + ArrayRef contents = sec->getContents(); + if (!canUseDebugH(contents)) + return None; + return contents; +} + +static ArrayRef +getHashesFromDebugH(ArrayRef debugH) { + assert(canUseDebugH(debugH)); + + debugH = debugH.drop_front(sizeof(object::debug_h_header)); + uint32_t count = debugH.size() / sizeof(GloballyHashedType); + return {reinterpret_cast(debugH.data()), count}; +} + +// Merge .debug$T for a generic object file. +Expected TpiSource::mergeDebugT(TypeMerger *m, + CVIndexMap *indexMap) { + CVTypeArray types; + BinaryStreamReader reader(file->debugTypes, support::little); + cantFail(reader.readArray(types, reader.getLength())); + + if (config->debugGHashes) { + ArrayRef hashes; + std::vector ownedHashes; + if (Optional> debugH = getDebugH(file)) + hashes = getHashesFromDebugH(*debugH); + else { + ownedHashes = GloballyHashedType::hashTypes(types); + hashes = ownedHashes; + } + + if (auto err = mergeTypeAndIdRecords(m->globalIDTable, m->globalTypeTable, + indexMap->tpiMap, types, hashes, + file->pchSignature)) + fatal("codeview::mergeTypeAndIdRecords failed: " + + toString(std::move(err))); + } else { + if (auto err = + mergeTypeAndIdRecords(m->idTable, m->typeTable, indexMap->tpiMap, + types, file->pchSignature)) + fatal("codeview::mergeTypeAndIdRecords failed: " + + toString(std::move(err))); + } + + if (config->showSummary) { + // Count how many times we saw each type record in our input. This + // calculation requires a second pass over the type records to classify each + // record as a type or index. This is slow, but this code executes when + // collecting statistics. + m->tpiCounts.resize(m->getTypeTable().size()); + m->ipiCounts.resize(m->getIDTable().size()); + uint32_t srcIdx = 0; + for (CVType &ty : types) { + TypeIndex dstIdx = indexMap->tpiMap[srcIdx++]; + // Type merging may fail, so a complex source type may become the simple + // NotTranslated type, which cannot be used as an array index. + if (dstIdx.isSimple()) + continue; + SmallVectorImpl &counts = + isIdRecord(ty.kind()) ? m->ipiCounts : m->tpiCounts; + ++counts[dstIdx.toArrayIndex()]; + } + } + + return indexMap; } -// If existing, return the actual PDB path on disk. -static Optional findPdbPath(StringRef pdbPath, - const ObjFile *dependentFile) { - // Ensure the file exists before anything else. In some cases, if the path - // points to a removable device, Driver::enqueuePath() would fail with an - // error (EAGAIN, "resource unavailable try again") which we want to skip - // silently. - if (llvm::sys::fs::exists(pdbPath)) - return normalizePdbPath(pdbPath); - std::string ret = getPdbBaseName(dependentFile, pdbPath); - if (llvm::sys::fs::exists(ret)) - return normalizePdbPath(ret); - return None; +// Merge types from a type server PDB. +Expected TypeServerSource::mergeDebugT(TypeMerger *m, + CVIndexMap *) { + pdb::PDBFile &pdbFile = pdbInputFile->session->getPDBFile(); + Expected expectedTpi = pdbFile.getPDBTpiStream(); + if (auto e = expectedTpi.takeError()) + fatal("Type server does not have TPI stream: " + toString(std::move(e))); + pdb::TpiStream *maybeIpi = nullptr; + if (pdbFile.hasPDBIpiStream()) { + Expected expectedIpi = pdbFile.getPDBIpiStream(); + if (auto e = expectedIpi.takeError()) + fatal("Error getting type server IPI stream: " + toString(std::move(e))); + maybeIpi = &*expectedIpi; + } + + if (config->debugGHashes) { + // PDBs do not actually store global hashes, so when merging a type server + // PDB we have to synthesize global hashes. To do this, we first synthesize + // global hashes for the TPI stream, since it is independent, then we + // synthesize hashes for the IPI stream, using the hashes for the TPI stream + // as inputs. + auto tpiHashes = GloballyHashedType::hashTypes(expectedTpi->typeArray()); + Optional endPrecomp; + // Merge TPI first, because the IPI stream will reference type indices. + if (auto err = + mergeTypeRecords(m->globalTypeTable, tsIndexMap.tpiMap, + expectedTpi->typeArray(), tpiHashes, endPrecomp)) + fatal("codeview::mergeTypeRecords failed: " + toString(std::move(err))); + + // Merge IPI. + if (maybeIpi) { + auto ipiHashes = + GloballyHashedType::hashIds(maybeIpi->typeArray(), tpiHashes); + if (auto err = mergeIdRecords(m->globalIDTable, tsIndexMap.tpiMap, + tsIndexMap.ipiMap, maybeIpi->typeArray(), + ipiHashes)) + fatal("codeview::mergeIdRecords failed: " + toString(std::move(err))); + } + } else { + // Merge TPI first, because the IPI stream will reference type indices. + if (auto err = mergeTypeRecords(m->typeTable, tsIndexMap.tpiMap, + expectedTpi->typeArray())) + fatal("codeview::mergeTypeRecords failed: " + toString(std::move(err))); + + // Merge IPI. + if (maybeIpi) { + if (auto err = mergeIdRecords(m->idTable, tsIndexMap.tpiMap, + tsIndexMap.ipiMap, maybeIpi->typeArray())) + fatal("codeview::mergeIdRecords failed: " + toString(std::move(err))); + } + } + + if (config->showSummary) { + // Count how many times we saw each type record in our input. If a + // destination type index is present in the source to destination type index + // map, that means we saw it once in the input. Add it to our histogram. + m->tpiCounts.resize(m->getTypeTable().size()); + m->ipiCounts.resize(m->getIDTable().size()); + for (TypeIndex ti : tsIndexMap.tpiMap) + if (!ti.isSimple()) + ++m->tpiCounts[ti.toArrayIndex()]; + for (TypeIndex ti : tsIndexMap.ipiMap) + if (!ti.isSimple()) + ++m->ipiCounts[ti.toArrayIndex()]; + } + + return &tsIndexMap; } -// Fetch the PDB instance that was already loaded by the COFF Driver. -Expected -TypeServerSource::findFromFile(const ObjFile *dependentFile) { - const TypeServer2Record &ts = - retrieveDependencyInfo(dependentFile->debugTypesObj); +Expected +UseTypeServerSource::mergeDebugT(TypeMerger *m, CVIndexMap *indexMap) { + const codeview::GUID &tsId = typeServerDependency.getGuid(); + StringRef tsPath = typeServerDependency.getName(); + + TypeServerSource *tsSrc; + auto it = TypeServerSource::mappings.find(tsId); + if (it != TypeServerSource::mappings.end()) { + tsSrc = it->second; + } else { + // The file failed to load, lookup by name + PDBInputFile *pdb = PDBInputFile::findFromRecordPath(tsPath, file); + if (!pdb) + return createFileError(tsPath, errorCodeToError(std::error_code( + ENOENT, std::generic_category()))); + // If an error occurred during loading, throw it now + if (pdb->loadErr && *pdb->loadErr) + return createFileError(tsPath, std::move(*pdb->loadErr)); + + tsSrc = (TypeServerSource *)pdb->debugTypesObj; + } + + pdb::PDBFile &pdbSession = tsSrc->pdbInputFile->session->getPDBFile(); + auto expectedInfo = pdbSession.getPDBInfoStream(); + if (!expectedInfo) + return &tsSrc->tsIndexMap; + + // Just because a file with a matching name was found and it was an actual + // PDB file doesn't mean it matches. For it to match the InfoStream's GUID + // must match the GUID specified in the TypeServer2 record. + if (expectedInfo->getGuid() != typeServerDependency.getGuid()) + return createFileError( + tsPath, + make_error(pdb::pdb_error_code::signature_out_of_date)); - Optional p = findPdbPath(ts.Name, dependentFile); - if (!p) - return createFileError(ts.Name, errorCodeToError(std::error_code( - ENOENT, std::generic_category()))); + return &tsSrc->tsIndexMap; +} - auto it = TypeServerSource::instances.find(*p); - // The PDB file exists on disk, at this point we expect it to have been - // inserted in the map by TypeServerSource::loadPDB() - assert(it != TypeServerSource::instances.end()); +static bool equalsPath(StringRef path1, StringRef path2) { +#if defined(_WIN32) + return path1.equals_lower(path2); +#else + return path1.equals(path2); +#endif +} - std::pair &pdb = it->second; +// Find by name an OBJ provided on the command line +static PrecompSource *findObjByName(StringRef fileNameOnly) { + SmallString<128> currentPath; + for (auto kv : PrecompSource::mappings) { + StringRef currentFileName = sys::path::filename(kv.second->file->getName(), + sys::path::Style::windows); + + // Compare based solely on the file name (link.exe behavior) + if (equalsPath(currentFileName, fileNameOnly)) + return kv.second; + } + return nullptr; +} - if (!pdb.second) +Expected findPrecompMap(ObjFile *file, PrecompRecord &pr) { + // Cross-compile warning: given that Clang doesn't generate LF_PRECOMP + // records, we assume the OBJ comes from a Windows build of cl.exe. Thusly, + // the paths embedded in the OBJs are in the Windows format. + SmallString<128> prFileName = + sys::path::filename(pr.getPrecompFilePath(), sys::path::Style::windows); + + PrecompSource *precomp; + auto it = PrecompSource::mappings.find(pr.getSignature()); + if (it != PrecompSource::mappings.end()) { + precomp = it->second; + } else { + // Lookup by name + precomp = findObjByName(prFileName); + } + + if (!precomp) return createFileError( - *p, createStringError(inconvertibleErrorCode(), pdb.first.c_str())); + prFileName, + make_error(pdb::pdb_error_code::no_matching_pch)); - pdb::PDBFile &pdbFile = (pdb.second)->session->getPDBFile(); - pdb::InfoStream &info = cantFail(pdbFile.getPDBInfoStream()); + if (pr.getSignature() != file->pchSignature) + return createFileError( + toString(file), + make_error(pdb::pdb_error_code::no_matching_pch)); - // Just because a file with a matching name was found doesn't mean it can be - // used. The GUID must match between the PDB header and the OBJ - // TypeServer2 record. The 'Age' is used by MSVC incremental compilation. - if (info.getGuid() != ts.getGuid()) + if (pr.getSignature() != *precomp->file->pchSignature) return createFileError( - ts.Name, - make_error(pdb::pdb_error_code::signature_out_of_date)); + toString(precomp->file), + make_error(pdb::pdb_error_code::no_matching_pch)); + + return &precomp->precompIndexMap; +} + +/// Merges a precompiled headers TPI map into the current TPI map. The +/// precompiled headers object will also be loaded and remapped in the +/// process. +static Expected +mergeInPrecompHeaderObj(ObjFile *file, CVIndexMap *indexMap, + PrecompRecord &precomp) { + auto e = findPrecompMap(file, precomp); + if (!e) + return e.takeError(); + + const CVIndexMap *precompIndexMap = *e; + assert(precompIndexMap->isPrecompiledTypeMap); + + if (precompIndexMap->tpiMap.empty()) + return precompIndexMap; + + assert(precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex); + assert(precomp.getTypesCount() <= precompIndexMap->tpiMap.size()); + // Use the previously remapped index map from the precompiled headers. + indexMap->tpiMap.append(precompIndexMap->tpiMap.begin(), + precompIndexMap->tpiMap.begin() + + precomp.getTypesCount()); + return indexMap; +} - return pdb.second; +Expected +UsePrecompSource::mergeDebugT(TypeMerger *m, CVIndexMap *indexMap) { + // This object was compiled with /Yu, so process the corresponding + // precompiled headers object (/Yc) first. Some type indices in the current + // object are referencing data in the precompiled headers object, so we need + // both to be loaded. + auto e = mergeInPrecompHeaderObj(file, indexMap, precompDependency); + if (!e) + return e.takeError(); + + // Drop LF_PRECOMP record from the input stream, as it has been replaced + // with the precompiled headers Type stream in the mergeInPrecompHeaderObj() + // call above. Note that we can't just call Types.drop_front(), as we + // explicitly want to rebase the stream. + CVTypeArray types; + BinaryStreamReader reader(file->debugTypes, support::little); + cantFail(reader.readArray(types, reader.getLength())); + auto firstType = types.begin(); + file->debugTypes = file->debugTypes.drop_front(firstType->RecordData.size()); + + return TpiSource::mergeDebugT(m, indexMap); } -// FIXME: Temporary interface until PDBLinker::maybeMergeTypeServerPDB() is -// moved here. -Expected -lld::coff::findTypeServerSource(const ObjFile *f) { - Expected ts = TypeServerSource::findFromFile(f); - if (!ts) - return ts.takeError(); - return ts.get()->session.get(); +Expected PrecompSource::mergeDebugT(TypeMerger *m, + CVIndexMap *) { + // Note that we're not using the provided CVIndexMap. Instead, we use our + // local one. Precompiled headers objects need to save the index map for + // further reference by other objects which use the precompiled headers. + return TpiSource::mergeDebugT(m, &precompIndexMap); } -// Queue a PDB type server for loading in the COFF Driver -void TypeServerSource::enqueue(const ObjFile *dependentFile, - const TypeServer2Record &ts) { - // Start by finding where the PDB is located (either the record path or next - // to the OBJ file) - Optional p = findPdbPath(ts.Name, dependentFile); - if (!p) - return; - auto it = TypeServerSource::instances.emplace( - *p, std::pair{}); - if (!it.second) - return; // another OBJ already scheduled this PDB for load - - driver->enqueuePath(*p, false, false); +uint32_t TpiSource::countTypeServerPDBs() { + return TypeServerSource::mappings.size(); } -// Create an instance of TypeServerSource or an error string if the PDB couldn't -// be loaded. The error message will be displayed later, when the referring OBJ -// will be merged in. NOTE - a PDB load failure is not a link error: some -// debug info will simply be missing from the final PDB - that is the default -// accepted behavior. -void lld::coff::loadTypeServerSource(llvm::MemoryBufferRef m) { - std::string path = normalizePdbPath(m.getBufferIdentifier()); - - Expected ts = TypeServerSource::getInstance(m); - if (!ts) - TypeServerSource::instances[path] = {toString(ts.takeError()), nullptr}; - else - TypeServerSource::instances[path] = {{}, *ts}; +uint32_t TpiSource::countPrecompObjs() { + return PrecompSource::mappings.size(); } -Expected TypeServerSource::getInstance(MemoryBufferRef m) { - std::unique_ptr iSession; - Error err = pdb::NativeSession::createFromPdb( - MemoryBuffer::getMemBuffer(m, false), iSession); - if (err) - return std::move(err); - - std::unique_ptr session( - static_cast(iSession.release())); - - pdb::PDBFile &pdbFile = session->getPDBFile(); - Expected info = pdbFile.getPDBInfoStream(); - // All PDB Files should have an Info stream. - if (!info) - return info.takeError(); - return make(m, session.release()); +void TpiSource::clear() { + gc.clear(); + TypeServerSource::mappings.clear(); + PrecompSource::mappings.clear(); } diff --git a/lld/COFF/Driver.h b/lld/COFF/Driver.h --- a/lld/COFF/Driver.h +++ b/lld/COFF/Driver.h @@ -87,6 +87,8 @@ void enqueueArchiveMember(const Archive::Child &c, const Archive::Symbol &sym, StringRef parentName); + void enqueuePDB(StringRef Path) { enqueuePath(Path, false, false); } + MemoryBufferRef takeBuffer(std::unique_ptr mb); void enqueuePath(StringRef path, bool wholeArchive, bool lazy); diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -89,6 +89,8 @@ ImportFile::instances.clear(); BitcodeFile::instances.clear(); memset(MergeChunk::instances, 0, sizeof(MergeChunk::instances)); + TpiSource::clear(); + return !errorCount(); } @@ -218,7 +220,7 @@ symtab->addFile(make(mbref)); break; case file_magic::pdb: - loadTypeServerSource(mbref); + symtab->addFile(make(mbref)); break; case file_magic::coff_cl_gl_object: error(filename + ": is not a native COFF file. Recompile without /GL"); diff --git a/lld/COFF/InputFiles.h b/lld/COFF/InputFiles.h --- a/lld/COFF/InputFiles.h +++ b/lld/COFF/InputFiles.h @@ -26,6 +26,7 @@ struct DILineInfo; namespace pdb { class DbiModuleDescriptorBuilder; +class NativeSession; } namespace lto { class InputFile; @@ -64,6 +65,7 @@ ArchiveKind, ObjectKind, LazyObjectKind, + PDBKind, ImportKind, BitcodeKind }; @@ -299,6 +301,32 @@ DWARFCache *dwarf = nullptr; }; +// This is a PDB type server dependency, that is not a input file per se, but +// needs to be treated like one. Such files are discovered from the debug type +// stream. +class PDBInputFile : public InputFile { +public: + explicit PDBInputFile(MemoryBufferRef m); + ~PDBInputFile(); + static bool classof(const InputFile *f) { return f->kind() == PDBKind; } + void parse() override; + + static void enqueue(StringRef path, ObjFile *fromFile); + + static PDBInputFile *findFromRecordPath(StringRef path, ObjFile *fromFile); + + static std::map instances; + + // Record possible errors while opening the PDB file + llvm::Optional loadErr; + + // This is the actual interface to the PDB (if it was opened successfully) + std::unique_ptr session; + + // If the PDB has a .debug$T stream, this tells how it will be handled. + TpiSource *debugTypesObj = nullptr; +}; + // This type represents import library members that contain DLL names // and symbols exported from the DLLs. See Microsoft PE/COFF spec. 7 // for details about the format. diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp --- a/lld/COFF/InputFiles.cpp +++ b/lld/COFF/InputFiles.cpp @@ -25,6 +25,8 @@ #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/TypeDeserializer.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/Native/PDBFile.h" #include "llvm/LTO/LTO.h" #include "llvm/Object/Binary.h" #include "llvm/Object/COFF.h" @@ -68,6 +70,7 @@ } std::vector ObjFile::instances; +std::map PDBInputFile::instances; std::vector ImportFile::instances; std::vector BitcodeFile::instances; @@ -755,10 +758,11 @@ if (data.empty()) return; + // Get the first type record. It will indicate if this object uses a type + // server (/Zi) or a PCH file (/Yu). CVTypeArray types; BinaryStreamReader reader(data, support::little); cantFail(reader.readArray(types, reader.getLength())); - CVTypeArray::Iterator firstType = types.begin(); if (firstType == types.end()) return; @@ -766,28 +770,120 @@ // Remember the .debug$T or .debug$P section. debugTypes = data; + // This object file is a PCH file that others will depend on. if (isPCH) { debugTypesObj = makePrecompSource(this); return; } + // This object file was compiled with /Zi. Enqueue the PDB dependency. if (firstType->kind() == LF_TYPESERVER2) { TypeServer2Record ts = cantFail( TypeDeserializer::deserializeAs(firstType->data())); - debugTypesObj = makeUseTypeServerSource(this, &ts); + debugTypesObj = makeUseTypeServerSource(this, ts); + PDBInputFile::enqueue(ts.getName(), this); return; } + // This object was compiled with /Yu. It uses types from another object file + // with a matching signature. if (firstType->kind() == LF_PRECOMP) { PrecompRecord precomp = cantFail( TypeDeserializer::deserializeAs(firstType->data())); - debugTypesObj = makeUsePrecompSource(this, &precomp); + debugTypesObj = makeUsePrecompSource(this, precomp); return; } + // This is a plain old object file. debugTypesObj = makeTpiSource(this); } +// Make a PDB path assuming the PDB is in the same folder as the OBJ +static std::string getPdbBaseName(ObjFile *file, StringRef tSPath) { + StringRef localPath = + !file->parentName.empty() ? file->parentName : file->getName(); + SmallString<128> path = sys::path::parent_path(localPath); + + // Currently, type server PDBs are only created by MSVC cl, which only runs + // on Windows, so we can assume type server paths are Windows style. + sys::path::append(path, + sys::path::filename(tSPath, sys::path::Style::windows)); + return std::string(path.str()); +} + +// The casing of the PDB path stamped in the OBJ can differ from the actual path +// on disk. With this, we ensure to always use lowercase as a key for the +// PDBInputFile::instances map, at least on Windows. +static std::string normalizePdbPath(StringRef path) { +#if defined(_WIN32) + return path.lower(); +#else // LINUX + return std::string(path); +#endif +} + +// If existing, return the actual PDB path on disk. +static Optional findPdbPath(StringRef pdbPath, + ObjFile *dependentFile) { + // Ensure the file exists before anything else. In some cases, if the path + // points to a removable device, Driver::enqueuePath() would fail with an + // error (EAGAIN, "resource unavailable try again") which we want to skip + // silently. + if (llvm::sys::fs::exists(pdbPath)) + return normalizePdbPath(pdbPath); + std::string ret = getPdbBaseName(dependentFile, pdbPath); + if (llvm::sys::fs::exists(ret)) + return normalizePdbPath(ret); + return None; +} + +PDBInputFile::PDBInputFile(MemoryBufferRef m) : InputFile(PDBKind, m) {} + +PDBInputFile::~PDBInputFile() = default; + +PDBInputFile *PDBInputFile::findFromRecordPath(StringRef path, + ObjFile *fromFile) { + auto p = findPdbPath(path.str(), fromFile); + if (!p) + return nullptr; + auto it = PDBInputFile::instances.find(*p); + if (it != PDBInputFile::instances.end()) + return it->second; + return nullptr; +} + +void PDBInputFile::enqueue(StringRef path, ObjFile *fromFile) { + auto p = findPdbPath(path.str(), fromFile); + if (!p) + return; + auto it = PDBInputFile::instances.emplace(*p, nullptr); + if (!it.second) + return; // already scheduled for load + driver->enqueuePDB(*p); +} + +void PDBInputFile::parse() { + PDBInputFile::instances[mb.getBufferIdentifier().str()] = this; + + std::unique_ptr thisSession; + loadErr.emplace(pdb::NativeSession::createFromPdb( + MemoryBuffer::getMemBuffer(mb, false), thisSession)); + if (*loadErr) + return; // fail silently at this point - the error will be handled later, + // when merging the debug type stream + + session.reset(static_cast(thisSession.release())); + + pdb::PDBFile &pdbFile = session->getPDBFile(); + auto expectedInfo = pdbFile.getPDBInfoStream(); + // All PDB Files should have an Info stream. + if (!expectedInfo) { + loadErr.emplace(expectedInfo.takeError()); + return; + } + debugTypesObj = makeTypeServerSource(this); +} + // Used only for DWARF debug info, which is not common (except in MinGW // environments). This returns an optional pair of file name and line // number for where the variable was defined. diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp --- a/lld/COFF/PDB.cpp +++ b/lld/COFF/PDB.cpp @@ -26,11 +26,7 @@ #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h" #include "llvm/DebugInfo/CodeView/SymbolRecordHelpers.h" #include "llvm/DebugInfo/CodeView/SymbolSerializer.h" -#include "llvm/DebugInfo/CodeView/TypeDeserializer.h" -#include "llvm/DebugInfo/CodeView/TypeDumpVisitor.h" #include "llvm/DebugInfo/CodeView/TypeIndexDiscovery.h" -#include "llvm/DebugInfo/CodeView/TypeRecordHelpers.h" -#include "llvm/DebugInfo/CodeView/TypeStreamMerger.h" #include "llvm/DebugInfo/MSF/MSFBuilder.h" #include "llvm/DebugInfo/MSF/MSFCommon.h" #include "llvm/DebugInfo/PDB/GenericError.h" @@ -114,44 +110,11 @@ /// Link CodeView from a single object file into the target (output) PDB. /// When a precompiled headers object is linked, its TPI map might be provided /// externally. - void addObjFile(ObjFile *file, CVIndexMap *externIndexMap = nullptr); - - /// Produce a mapping from the type and item indices used in the object - /// file to those in the destination PDB. - /// - /// If the object file uses a type server PDB (compiled with /Zi), merge TPI - /// and IPI from the type server PDB and return a map for it. Each unique type - /// server PDB is merged at most once, so this may return an existing index - /// mapping. - /// - /// If the object does not use a type server PDB (compiled with /Z7), we merge - /// all the type and item records from the .debug$S stream and fill in the - /// caller-provided objectIndexMap. - Expected mergeDebugT(ObjFile *file, - CVIndexMap *objectIndexMap); - - /// Reads and makes available a PDB. - Expected maybeMergeTypeServerPDB(ObjFile *file); - - /// Merges a precompiled headers TPI map into the current TPI map. The - /// precompiled headers object will also be loaded and remapped in the - /// process. - Error mergeInPrecompHeaderObj(ObjFile *file, CVIndexMap *objectIndexMap); - - /// Reads and makes available a precompiled headers object. - /// - /// This is a requirement for objects compiled with cl.exe /Yu. In that - /// case, the referenced object (which was compiled with /Yc) has to be loaded - /// first. This is mainly because the current object's TPI stream has external - /// references to the precompiled headers object. - /// - /// If the precompiled headers object was already loaded, this function will - /// simply return its (remapped) TPI map. - Expected aquirePrecompObj(ObjFile *file); - - /// Adds a precompiled headers object signature -> TPI mapping. - std::pair - registerPrecompiledHeaders(uint32_t signature); + void addDebug(TpiSource *source); + + const CVIndexMap *mergeTypeRecords(TpiSource *source, CVIndexMap *localMap); + + void addDebugSymbols(ObjFile *file, const CVIndexMap *indexMap); void mergeSymbolRecords(ObjFile *file, const CVIndexMap &indexMap, std::vector &stringTableRefs, @@ -180,22 +143,10 @@ llvm::SmallString<128> nativePath; - /// Type index mappings of type server PDBs that we've loaded so far. - std::map typeServerIndexMappings; - - /// Type index mappings of precompiled objects type map that we've loaded so - /// far. - std::map precompTypeIndexMappings; - // For statistics uint64_t globalSymbols = 0; uint64_t moduleSymbols = 0; uint64_t publicSymbols = 0; - - // When showSummary is enabled, these are histograms of TPI and IPI records - // keyed by type index. - SmallVector tpiCounts; - SmallVector ipiCounts; }; class DebugSHandler { @@ -205,7 +156,7 @@ ObjFile &file; /// The result of merging type indices. - const CVIndexMap &indexMap; + const CVIndexMap *indexMap; /// The DEBUG_S_STRINGTABLE subsection. These strings are referred to by /// index from other records in the .debug$S section. All of these strings @@ -239,7 +190,7 @@ std::vector stringTableReferences; public: - DebugSHandler(PDBLinker &linker, ObjFile &file, const CVIndexMap &indexMap) + DebugSHandler(PDBLinker &linker, ObjFile &file, const CVIndexMap *indexMap) : linker(linker), file(file), indexMap(indexMap) {} void handleDebugS(lld::coff::SectionChunk &debugS); @@ -290,42 +241,6 @@ fileName = std::move(absoluteFileName); } -// A COFF .debug$H section is currently a clang extension. This function checks -// if a .debug$H section is in a format that we expect / understand, so that we -// can ignore any sections which are coincidentally also named .debug$H but do -// not contain a format we recognize. -static bool canUseDebugH(ArrayRef debugH) { - if (debugH.size() < sizeof(object::debug_h_header)) - return false; - auto *header = - reinterpret_cast(debugH.data()); - debugH = debugH.drop_front(sizeof(object::debug_h_header)); - return header->Magic == COFF::DEBUG_HASHES_SECTION_MAGIC && - header->Version == 0 && - header->HashAlgorithm == uint16_t(GlobalTypeHashAlg::SHA1_8) && - (debugH.size() % 8 == 0); -} - -static Optional> getDebugH(ObjFile *file) { - SectionChunk *sec = - SectionChunk::findByName(file->getDebugChunks(), ".debug$H"); - if (!sec) - return llvm::None; - ArrayRef contents = sec->getContents(); - if (!canUseDebugH(contents)) - return None; - return contents; -} - -static ArrayRef -getHashesFromDebugH(ArrayRef debugH) { - assert(canUseDebugH(debugH)); - - debugH = debugH.drop_front(sizeof(object::debug_h_header)); - uint32_t count = debugH.size() / sizeof(GloballyHashedType); - return {reinterpret_cast(debugH.data()), count}; -} - static void addTypeInfo(pdb::TpiStreamBuilder &tpiBuilder, TypeCollection &typeTable) { // Start the TPI or IPI stream header. @@ -340,281 +255,6 @@ }); } -Expected -PDBLinker::mergeDebugT(ObjFile *file, CVIndexMap *objectIndexMap) { - ScopedTimer t(typeMergingTimer); - - if (!file->debugTypesObj) - return *objectIndexMap; // no Types stream - - // Precompiled headers objects need to save the index map for further - // reference by other objects which use the precompiled headers. - if (file->debugTypesObj->kind == TpiSource::PCH) { - uint32_t pchSignature = file->pchSignature.getValueOr(0); - if (pchSignature == 0) - fatal("No signature found for the precompiled headers OBJ (" + - file->getName() + ")"); - - // When a precompiled headers object comes first on the command-line, we - // update the mapping here. Otherwise, if an object referencing the - // precompiled headers object comes first, the mapping is created in - // aquirePrecompObj(), thus we would skip this block. - if (!objectIndexMap->isPrecompiledTypeMap) { - auto r = registerPrecompiledHeaders(pchSignature); - if (r.second) - fatal( - "A precompiled headers OBJ with the same signature was already " - "provided! (" + - file->getName() + ")"); - - objectIndexMap = &r.first; - } - } - - if (file->debugTypesObj->kind == TpiSource::UsingPDB) { - // Look through type servers. If we've already seen this type server, - // don't merge any type information. - return maybeMergeTypeServerPDB(file); - } - - CVTypeArray types; - BinaryStreamReader reader(file->debugTypes, support::little); - cantFail(reader.readArray(types, reader.getLength())); - - if (file->debugTypesObj->kind == TpiSource::UsingPCH) { - // This object was compiled with /Yu, so process the corresponding - // precompiled headers object (/Yc) first. Some type indices in the current - // object are referencing data in the precompiled headers object, so we need - // both to be loaded. - Error e = mergeInPrecompHeaderObj(file, objectIndexMap); - if (e) - return std::move(e); - - // Drop LF_PRECOMP record from the input stream, as it has been replaced - // with the precompiled headers Type stream in the mergeInPrecompHeaderObj() - // call above. Note that we can't just call Types.drop_front(), as we - // explicitly want to rebase the stream. - CVTypeArray::Iterator firstType = types.begin(); - types.setUnderlyingStream( - types.getUnderlyingStream().drop_front(firstType->RecordData.size())); - } - - // Fill in the temporary, caller-provided ObjectIndexMap. - if (config->debugGHashes) { - ArrayRef hashes; - std::vector ownedHashes; - if (Optional> debugH = getDebugH(file)) - hashes = getHashesFromDebugH(*debugH); - else { - ownedHashes = GloballyHashedType::hashTypes(types); - hashes = ownedHashes; - } - - if (auto err = mergeTypeAndIdRecords( - tMerger.globalIDTable, tMerger.globalTypeTable, - objectIndexMap->tpiMap, types, hashes, file->pchSignature)) - fatal("codeview::mergeTypeAndIdRecords failed: " + - toString(std::move(err))); - } else { - if (auto err = mergeTypeAndIdRecords(tMerger.idTable, tMerger.typeTable, - objectIndexMap->tpiMap, types, - file->pchSignature)) - fatal("codeview::mergeTypeAndIdRecords failed: " + - toString(std::move(err))); - } - - if (config->showSummary) { - // Count how many times we saw each type record in our input. This - // calculation requires a second pass over the type records to classify each - // record as a type or index. This is slow, but this code executes when - // collecting statistics. - tpiCounts.resize(tMerger.getTypeTable().size()); - ipiCounts.resize(tMerger.getIDTable().size()); - uint32_t srcIdx = 0; - for (CVType &ty : types) { - TypeIndex dstIdx = objectIndexMap->tpiMap[srcIdx++]; - // Type merging may fail, so a complex source type may become the simple - // NotTranslated type, which cannot be used as an array index. - if (dstIdx.isSimple()) - continue; - SmallVectorImpl &counts = - isIdRecord(ty.kind()) ? ipiCounts : tpiCounts; - ++counts[dstIdx.toArrayIndex()]; - } - } - - return *objectIndexMap; -} - -Expected PDBLinker::maybeMergeTypeServerPDB(ObjFile *file) { - Expected pdbSession = findTypeServerSource(file); - if (!pdbSession) - return pdbSession.takeError(); - - pdb::PDBFile &pdbFile = pdbSession.get()->getPDBFile(); - pdb::InfoStream &info = cantFail(pdbFile.getPDBInfoStream()); - - auto it = typeServerIndexMappings.emplace(info.getGuid(), CVIndexMap()); - CVIndexMap &indexMap = it.first->second; - if (!it.second) - return indexMap; // already merged - - // Mark this map as a type server map. - indexMap.isTypeServerMap = true; - - Expected expectedTpi = pdbFile.getPDBTpiStream(); - if (auto e = expectedTpi.takeError()) - fatal("Type server does not have TPI stream: " + toString(std::move(e))); - pdb::TpiStream *maybeIpi = nullptr; - if (pdbFile.hasPDBIpiStream()) { - Expected expectedIpi = pdbFile.getPDBIpiStream(); - if (auto e = expectedIpi.takeError()) - fatal("Error getting type server IPI stream: " + toString(std::move(e))); - maybeIpi = &*expectedIpi; - } - - if (config->debugGHashes) { - // PDBs do not actually store global hashes, so when merging a type server - // PDB we have to synthesize global hashes. To do this, we first synthesize - // global hashes for the TPI stream, since it is independent, then we - // synthesize hashes for the IPI stream, using the hashes for the TPI stream - // as inputs. - auto tpiHashes = GloballyHashedType::hashTypes(expectedTpi->typeArray()); - Optional endPrecomp; - // Merge TPI first, because the IPI stream will reference type indices. - if (auto err = - mergeTypeRecords(tMerger.globalTypeTable, indexMap.tpiMap, - expectedTpi->typeArray(), tpiHashes, endPrecomp)) - fatal("codeview::mergeTypeRecords failed: " + toString(std::move(err))); - - // Merge IPI. - if (maybeIpi) { - auto ipiHashes = - GloballyHashedType::hashIds(maybeIpi->typeArray(), tpiHashes); - if (auto err = - mergeIdRecords(tMerger.globalIDTable, indexMap.tpiMap, - indexMap.ipiMap, maybeIpi->typeArray(), ipiHashes)) - fatal("codeview::mergeIdRecords failed: " + toString(std::move(err))); - } - } else { - // Merge TPI first, because the IPI stream will reference type indices. - if (auto err = mergeTypeRecords(tMerger.typeTable, indexMap.tpiMap, - expectedTpi->typeArray())) - fatal("codeview::mergeTypeRecords failed: " + toString(std::move(err))); - - // Merge IPI. - if (maybeIpi) { - if (auto err = mergeIdRecords(tMerger.idTable, indexMap.tpiMap, - indexMap.ipiMap, maybeIpi->typeArray())) - fatal("codeview::mergeIdRecords failed: " + toString(std::move(err))); - } - } - - if (config->showSummary) { - // Count how many times we saw each type record in our input. If a - // destination type index is present in the source to destination type index - // map, that means we saw it once in the input. Add it to our histogram. - tpiCounts.resize(tMerger.getTypeTable().size()); - ipiCounts.resize(tMerger.getIDTable().size()); - for (TypeIndex ti : indexMap.tpiMap) - if (!ti.isSimple()) - ++tpiCounts[ti.toArrayIndex()]; - for (TypeIndex ti : indexMap.ipiMap) - if (!ti.isSimple()) - ++ipiCounts[ti.toArrayIndex()]; - } - - return indexMap; -} - -Error PDBLinker::mergeInPrecompHeaderObj(ObjFile *file, - CVIndexMap *objectIndexMap) { - const PrecompRecord &precomp = - retrieveDependencyInfo(file->debugTypesObj); - - Expected e = aquirePrecompObj(file); - if (!e) - return e.takeError(); - - const CVIndexMap &precompIndexMap = *e; - assert(precompIndexMap.isPrecompiledTypeMap); - - if (precompIndexMap.tpiMap.empty()) - return Error::success(); - - assert(precomp.getStartTypeIndex() == TypeIndex::FirstNonSimpleIndex); - assert(precomp.getTypesCount() <= precompIndexMap.tpiMap.size()); - // Use the previously remapped index map from the precompiled headers. - objectIndexMap->tpiMap.append(precompIndexMap.tpiMap.begin(), - precompIndexMap.tpiMap.begin() + - precomp.getTypesCount()); - return Error::success(); -} - -static bool equals_path(StringRef path1, StringRef path2) { -#if defined(_WIN32) - return path1.equals_lower(path2); -#else - return path1.equals(path2); -#endif -} -// Find by name an OBJ provided on the command line -static ObjFile *findObjWithPrecompSignature(StringRef fileNameOnly, - uint32_t precompSignature) { - for (ObjFile *f : ObjFile::instances) { - StringRef currentFileName = sys::path::filename(f->getName()); - - if (f->pchSignature.hasValue() && - f->pchSignature.getValue() == precompSignature && - equals_path(fileNameOnly, currentFileName)) - return f; - } - return nullptr; -} - -std::pair -PDBLinker::registerPrecompiledHeaders(uint32_t signature) { - auto insertion = precompTypeIndexMappings.insert({signature, CVIndexMap()}); - CVIndexMap &indexMap = insertion.first->second; - if (!insertion.second) - return {indexMap, true}; - // Mark this map as a precompiled types map. - indexMap.isPrecompiledTypeMap = true; - return {indexMap, false}; -} - -Expected PDBLinker::aquirePrecompObj(ObjFile *file) { - const PrecompRecord &precomp = - retrieveDependencyInfo(file->debugTypesObj); - - // First, check if we already loaded the precompiled headers object with this - // signature. Return the type index mapping if we've already seen it. - auto r = registerPrecompiledHeaders(precomp.getSignature()); - if (r.second) - return r.first; - - CVIndexMap &indexMap = r.first; - - // Cross-compile warning: given that Clang doesn't generate LF_PRECOMP - // records, we assume the OBJ comes from a Windows build of cl.exe. Thusly, - // the paths embedded in the OBJs are in the Windows format. - SmallString<128> precompFileName = sys::path::filename( - precomp.getPrecompFilePath(), sys::path::Style::windows); - - // link.exe requires that a precompiled headers object must always be provided - // on the command-line, even if that's not necessary. - auto precompFile = - findObjWithPrecompSignature(precompFileName, precomp.Signature); - if (!precompFile) - return createFileError( - precomp.getPrecompFilePath().str(), - make_error(pdb::pdb_error_code::no_matching_pch)); - - addObjFile(precompFile, &indexMap); - - return indexMap; -} - static bool remapTypeIndex(TypeIndex &ti, ArrayRef typeIndexMap) { if (ti.isSimple()) return true; @@ -1040,6 +680,11 @@ BinaryStreamReader reader(relocatedDebugContents, support::little); exitOnErr(reader.readArray(subsections, relocatedDebugContents.size())); + // If there is no index map, use an empty one. + CVIndexMap tempIndexMap; + if (!indexMap) + indexMap = &tempIndexMap; + for (const DebugSubsectionRecord &ss : subsections) { // Ignore subsections with the 'ignore' bit. Some versions of the Visual C++ // runtime have subsections with this bit set. @@ -1078,7 +723,7 @@ break; } case DebugSubsectionKind::Symbols: { - linker.mergeSymbolRecords(&file, indexMap, stringTableReferences, + linker.mergeSymbolRecords(&file, *indexMap, stringTableReferences, ss.getRecordData()); break; } @@ -1129,7 +774,7 @@ uint32_t sourceLine = line.Header->SourceLineNum; ArrayRef typeOrItemMap = - indexMap.isTypeServerMap ? indexMap.ipiMap : indexMap.tpiMap; + indexMap->isTypeServerMap ? indexMap->ipiMap : indexMap->tpiMap; if (!remapTypeIndex(inlinee, typeOrItemMap)) { log("ignoring inlinee line record in " + file.getName() + " with bad inlinee index 0x" + utohexstr(inlinee.getIndex())); @@ -1205,35 +850,39 @@ file.moduleDBI->addDebugSubsection(std::move(newChecksums)); } -void PDBLinker::addObjFile(ObjFile *file, CVIndexMap *externIndexMap) { - if (file->mergedIntoPDB) +static void warnUnusable(InputFile *f, Error e) { + if (!config->warnDebugInfoUnusable) { + consumeError(std::move(e)); return; - file->mergedIntoPDB = true; + } + auto msg = "Cannot use debug info for '" + toString(f) + "' [LNK4099]"; + if (e) + warn(msg + "\n>>> failed to load reference " + toString(std::move(e))); + else + warn(msg); +} +const CVIndexMap *PDBLinker::mergeTypeRecords(TpiSource *source, + CVIndexMap *localMap) { + ScopedTimer t(typeMergingTimer); // Before we can process symbol substreams from .debug$S, we need to process // type information, file checksums, and the string table. Add type info to // the PDB first, so that we can get the map from object file type and item // indices to PDB type and item indices. - CVIndexMap objectIndexMap; - auto indexMapResult = - mergeDebugT(file, externIndexMap ? externIndexMap : &objectIndexMap); + Expected r = source->mergeDebugT(&tMerger, localMap); // If the .debug$T sections fail to merge, assume there is no debug info. - if (!indexMapResult) { - if (!config->warnDebugInfoUnusable) { - consumeError(indexMapResult.takeError()); - return; - } - warn("Cannot use debug info for '" + toString(file) + "' [LNK4099]\n" + - ">>> failed to load reference " + - StringRef(toString(indexMapResult.takeError()))); - return; + if (!r) { + warnUnusable(source->file, r.takeError()); + return nullptr; } + return *r; +} +void PDBLinker::addDebugSymbols(ObjFile *file, const CVIndexMap *indexMap) { ScopedTimer t(symbolMergingTimer); - pdb::DbiStreamBuilder &dbiBuilder = builder.getDbiBuilder(); - DebugSHandler dsh(*this, *file, *indexMapResult); + DebugSHandler dsh(*this, *file, indexMap); // Now do all live .debug$S and .debug$F sections. for (SectionChunk *debugChunk : file->getDebugChunks()) { if (!debugChunk->live || debugChunk->getSize() == 0) @@ -1269,34 +918,41 @@ // path to the object into the PDB. If this is a plain object, we make its // path absolute. If it's an object in an archive, we make the archive path // absolute. -static void createModuleDBI(pdb::PDBFileBuilder &builder) { +static void createModuleDBI(pdb::PDBFileBuilder &builder, ObjFile *file) { pdb::DbiStreamBuilder &dbiBuilder = builder.getDbiBuilder(); SmallString<128> objName; - for (ObjFile *file : ObjFile::instances) { + bool inArchive = !file->parentName.empty(); + objName = inArchive ? file->parentName : file->getName(); + pdbMakeAbsolute(objName); + StringRef modName = inArchive ? file->getName() : StringRef(objName); - bool inArchive = !file->parentName.empty(); - objName = inArchive ? file->parentName : file->getName(); - pdbMakeAbsolute(objName); - StringRef modName = inArchive ? file->getName() : StringRef(objName); + file->moduleDBI = &exitOnErr(dbiBuilder.addModuleInfo(modName)); + file->moduleDBI->setObjFileName(objName); - file->moduleDBI = &exitOnErr(dbiBuilder.addModuleInfo(modName)); - file->moduleDBI->setObjFileName(objName); + ArrayRef chunks = file->getChunks(); + uint32_t modi = file->moduleDBI->getModuleIndex(); - ArrayRef chunks = file->getChunks(); - uint32_t modi = file->moduleDBI->getModuleIndex(); - - for (Chunk *c : chunks) { - auto *secChunk = dyn_cast(c); - if (!secChunk || !secChunk->live) - continue; - pdb::SectionContrib sc = createSectionContrib(secChunk, modi); - file->moduleDBI->setFirstSectionContrib(sc); - break; - } + for (Chunk *c : chunks) { + auto *secChunk = dyn_cast(c); + if (!secChunk || !secChunk->live) + continue; + pdb::SectionContrib sc = createSectionContrib(secChunk, modi); + file->moduleDBI->setFirstSectionContrib(sc); + break; } } +void PDBLinker::addDebug(TpiSource *source) { + CVIndexMap localMap; + const CVIndexMap *indexMap = mergeTypeRecords(source, &localMap); + + if (source->kind == TpiSource::PDB) + return; // No symbols in TypeServer PDBs + + addDebugSymbols(source->file, indexMap); +} + static pdb::BulkPublic createPublic(Defined *def) { pdb::BulkPublic pub; pub.Name = def->getName().data(); @@ -1323,10 +979,30 @@ void PDBLinker::addObjectsToPDB() { ScopedTimer t1(addObjectsTimer); - createModuleDBI(builder); + // Create module descriptors + for_each(ObjFile::instances, + [&](ObjFile *obj) { createModuleDBI(builder, obj); }); - for (ObjFile *file : ObjFile::instances) - addObjFile(file); + // Merge OBJs that do not have debug types + for_each(ObjFile::instances, [&](ObjFile *obj) { + if (obj->debugTypesObj) + return; + // Even if there're no types, still merge non-symbol .Debug$S and .Debug$F + // sections + addDebugSymbols(obj, nullptr); + }); + + // Merge dependencies + TpiSource::forEachSource([&](TpiSource *source) { + if (source->isDependency()) + addDebug(source); + }); + + // Merge regular and dependent OBJs + TpiSource::forEachSource([&](TpiSource *source) { + if (!source->isDependency()) + addDebug(source); + }); builder.getStringTableBuilder().setStrings(pdbStrTab); t1.stop(); @@ -1373,8 +1049,8 @@ print(ObjFile::instances.size(), "Input OBJ files (expanded from all cmd-line inputs)"); - print(typeServerIndexMappings.size(), "PDB type server dependencies"); - print(precompTypeIndexMappings.size(), "Precomp OBJ dependencies"); + print(TpiSource::countTypeServerPDBs(), "PDB type server dependencies"); + print(TpiSource::countPrecompObjs(), "Precomp OBJ dependencies"); print(tMerger.getTypeTable().size() + tMerger.getIDTable().size(), "Merged TPI records"); print(pdbStrTab.size(), "Output PDB strings"); @@ -1428,8 +1104,8 @@ } }; - printLargeInputTypeRecs("TPI", tpiCounts, tMerger.getTypeTable()); - printLargeInputTypeRecs("IPI", ipiCounts, tMerger.getIDTable()); + printLargeInputTypeRecs("TPI", tMerger.tpiCounts, tMerger.getTypeTable()); + printLargeInputTypeRecs("IPI", tMerger.ipiCounts, tMerger.getIDTable()); message(buffer); } diff --git a/lld/COFF/TypeMerger.h b/lld/COFF/TypeMerger.h --- a/lld/COFF/TypeMerger.h +++ b/lld/COFF/TypeMerger.h @@ -48,6 +48,11 @@ /// Item records that will go into the PDB IPI stream (for /DEBUG:GHASH) llvm::codeview::GlobalTypeTableBuilder globalIDTable; + + // When showSummary is enabled, these are histograms of TPI and IPI records + // keyed by type index. + SmallVector tpiCounts; + SmallVector ipiCounts; }; /// Map from type index and item index in a type server PDB to the @@ -62,4 +67,4 @@ } // namespace coff } // namespace lld -#endif \ No newline at end of file +#endif diff --git a/lld/test/COFF/precomp-link.test b/lld/test/COFF/precomp-link.test --- a/lld/test/COFF/precomp-link.test +++ b/lld/test/COFF/precomp-link.test @@ -6,7 +6,12 @@ RUN: lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-invalid.obj %S/Inputs/precomp.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE -RUN: not lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE-MISSING-PRECOMPOBJ +FIXME: The following RUN line should fail, regardless of whether debug info is +enabled or not. Normally this would result in an error due to missing _PchSym_ +references, but SymbolTable.cpp suppresses such errors. MSVC seems to have a +special case for those symbols and it emits the LNK2011 error. + +RUN: lld-link %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe /opt:ref /opt:icf 2>&1 | FileCheck %s -check-prefix FAILURE-MISSING-PRECOMPOBJ FAILURE: warning: Cannot use debug info for '{{.*}}precomp-invalid.obj' [LNK4099] FAILURE-NEXT: failed to load reference '{{.*}}precomp.obj': No matching precompiled header could be located. @@ -14,6 +19,34 @@ FAILURE-MISSING-PRECOMPOBJ: warning: Cannot use debug info for '{{.*}}precomp-a.obj' [LNK4099] FAILURE-MISSING-PRECOMPOBJ-NEXT: failed to load reference '{{.*}}precomp.obj': No matching precompiled header could be located. +Check that a PCH object file with a missing S_OBJNAME record results in an +error. Edit out this record from the yaml-ified object: + - Kind: S_OBJNAME + ObjNameSym: + Signature: 545589255 + ObjectName: 'F:\svn\lld\test\COFF\precomp\precomp.obj' + +RUN: obj2yaml %S/Inputs/precomp.obj | grep -v 'SectionData: *04000000' > %t/precomp.yaml +RUN: sed '/S_OBJNAME/,/ObjectName:/d' < %t/precomp.yaml > precomp-no-objname.yaml +RUN: sed 's/Signature: *545589255/Signature: 0/' < %t/precomp.yaml > precomp-zero-sig.yaml +RUN: yaml2obj precomp-no-objname.yaml -o %t/precomp-no-objname.obj +RUN: yaml2obj precomp-zero-sig.yaml -o %t/precomp-zero-sig.obj + +RUN: not lld-link %t/precomp-no-objname.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix FAILURE-NO-SIGNATURE + +RUN: not lld-link %t/precomp-zero-sig.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix FAILURE-NO-SIGNATURE + +FAILURE-NO-SIGNATURE: error: {{.*}}.obj claims to be a PCH object, but does not have a valid signature + +Check that two PCH objs with duplicate signatures are an error. + +RUN: cp %S/Inputs/precomp.obj %t/precomp-dup.obj + +RUN: not lld-link %S/Inputs/precomp.obj %t/precomp-dup.obj %S/Inputs/precomp-a.obj %S/Inputs/precomp-b.obj /nodefaultlib /entry:main /debug /pdb:%t.pdb /out:%t.exe 2>&1 | FileCheck %s -check-prefix FAILURE-DUP-SIGNATURE + +FAILURE-DUP-SIGNATURE: error: a PCH object with the same signature has already been provided ({{.*precomp.obj and .*precomp-dup.obj.*}}) + + CHECK: Types (TPI Stream) CHECK-NOT: LF_PRECOMP CHECK-NOT: LF_ENDPRECOMP