Index: COFF/DebugTypes.h =================================================================== --- COFF/DebugTypes.h +++ COFF/DebugTypes.h @@ -10,12 +10,16 @@ #define LLD_COFF_DEBUGTYPES_H #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" namespace llvm { namespace codeview { class PrecompRecord; class TypeServer2Record; } // namespace codeview +namespace pdb { +class NativeSession; +} } // namespace llvm namespace lld { @@ -27,25 +31,30 @@ public: enum TpiKind { Regular, PCH, UsingPCH, PDB, UsingPDB }; - TpiSource(TpiKind K, ObjFile *F); + TpiSource(TpiKind K, const ObjFile *F); virtual ~TpiSource() {} const TpiKind Kind; - ObjFile *File; + const ObjFile *File; }; -TpiSource *makeTpiSource(ObjFile *F); -TpiSource *makeTypeServerSource(ObjFile *F); -TpiSource *makeUseTypeServerSource(ObjFile *F, - llvm::codeview::TypeServer2Record *TS); -TpiSource *makePrecompSource(ObjFile *F); -TpiSource *makeUsePrecompSource(ObjFile *F, - llvm::codeview::PrecompRecord *Precomp); +TpiSource *makeTpiSource(const ObjFile *F); +TpiSource *makeUseTypeServerSource(const ObjFile *F, + const llvm::codeview::TypeServer2Record *TS); +TpiSource *makePrecompSource(const ObjFile *F); +TpiSource *makeUsePrecompSource(const ObjFile *F, + const llvm::codeview::PrecompRecord *Precomp); + +void loadTypeServerSource(llvm::MemoryBufferRef M); // Temporary interface to get the dependency -template const T &retrieveDependencyInfo(TpiSource *Source); +template const T &retrieveDependencyInfo(const TpiSource *Source); + +// Temporary interface until we move PDBLinker::maybeMergeTypeServerPDB here +llvm::Expected +findTypeServerSource(const ObjFile *F); } // namespace coff } // namespace lld #endif \ No newline at end of file Index: COFF/DebugTypes.cpp =================================================================== --- COFF/DebugTypes.cpp +++ COFF/DebugTypes.cpp @@ -7,8 +7,15 @@ //===----------------------------------------------------------------------===// #include "DebugTypes.h" +#include "Driver.h" #include "InputFiles.h" +#include "lld/Common/ErrorHandler.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/PDB/GenericError.h" +#include "llvm/DebugInfo/PDB/Native/InfoStream.h" +#include "llvm/DebugInfo/PDB/Native/NativeSession.h" +#include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/Support/Path.h" using namespace lld; using namespace lld::coff; @@ -16,14 +23,44 @@ using namespace llvm::codeview; namespace { +// The TypeServerSource class represents a PDB type server, a file referenced by +// OBJ files compiled with MSVC /Zi. A single PDB can be shared by several OBJ +// files, therefore there must be only once instance per OBJ lot. The file path +// is discovered from the dependent OBJ's debug type stream. The +// TypeServerSource object is then queued and loaded by the COFF Driver. The +// debug type stream for such PDB files will be merged first in the final PDB, +// before any dependent OBJ. class TypeServerSource : public TpiSource { public: - TypeServerSource(ObjFile *F) : TpiSource(PDB, F) {} + explicit TypeServerSource(MemoryBufferRef M, llvm::pdb::NativeSession *S) + : TpiSource(PDB, nullptr), Session(S), MB(M) {} + + // Queue a PDB type server for loading in the COFF Driver + static void enqueue(const ObjFile *DependentFile, + const TypeServer2Record &TS); + + // Create an instance + static Expected getInstance(MemoryBufferRef M); + + // Fetch the PDB instance loaded for a corresponding dependent OBJ. + static Expected + findFromFile(const ObjFile *DependentFile); + + static std::map> + Instances; + + // The interface to the PDB (if it was opened successfully) + std::unique_ptr Session; + +private: + MemoryBufferRef MB; }; +// This class represents the debug type stream of an OBJ file that depends on a +// PDB type server (see TypeServerSource). class UseTypeServerSource : public TpiSource { public: - UseTypeServerSource(ObjFile *F, TypeServer2Record *TS) + UseTypeServerSource(const ObjFile *F, const TypeServer2Record *TS) : TpiSource(UsingPDB, F), TypeServerDependency(*TS) {} // Information about the PDB type server dependency, that needs to be loaded @@ -31,14 +68,20 @@ TypeServer2Record TypeServerDependency; }; +// This class represents the debug type stream of a Microsoft precompiled +// headers OBJ (PCH OBJ). This OBJ kind needs to be merged first in the output +// PDB, before any other OBJs that depend on this. Note that only MSVC generate +// such files, clang does not. class PrecompSource : public TpiSource { public: - PrecompSource(ObjFile *F) : TpiSource(PCH, F) {} + PrecompSource(const ObjFile *F) : TpiSource(PCH, F) {} }; +// This class represents the debug type stream of an OBJ file that depends on a +// Microsoft precompiled headers OBJ (see PrecompSource). class UsePrecompSource : public TpiSource { public: - UsePrecompSource(ObjFile *F, PrecompRecord *Precomp) + UsePrecompSource(const ObjFile *F, const PrecompRecord *Precomp) : TpiSource(UsingPCH, F), PrecompDependency(*Precomp) {} // Information about the Precomp OBJ dependency, that needs to be loaded in @@ -49,40 +92,176 @@ static std::vector> GC; -TpiSource::TpiSource(TpiKind K, ObjFile *F) : Kind(K), File(F) { +TpiSource::TpiSource(TpiKind K, const ObjFile *F) : Kind(K), File(F) { GC.push_back(std::unique_ptr(this)); } -TpiSource *coff::makeTpiSource(ObjFile *F) { +TpiSource *lld::coff::makeTpiSource(const ObjFile *F) { return new TpiSource(TpiSource::Regular, F); } -TpiSource *coff::makeTypeServerSource(ObjFile *F) { - return new TypeServerSource(F); -} - -TpiSource *coff::makeUseTypeServerSource(ObjFile *F, TypeServer2Record *TS) { +TpiSource *lld::coff::makeUseTypeServerSource(const ObjFile *F, + const TypeServer2Record *TS) { + TypeServerSource::enqueue(F, *TS); return new UseTypeServerSource(F, TS); } -TpiSource *coff::makePrecompSource(ObjFile *F) { return new PrecompSource(F); } +TpiSource *lld::coff::makePrecompSource(const ObjFile *F) { + return new PrecompSource(F); +} -TpiSource *coff::makeUsePrecompSource(ObjFile *F, PrecompRecord *Precomp) { +TpiSource *lld::coff::makeUsePrecompSource(const ObjFile *F, + const PrecompRecord *Precomp) { return new UsePrecompSource(F, Precomp); } namespace lld { namespace coff { template <> -const PrecompRecord &retrieveDependencyInfo(TpiSource *Source) { +const PrecompRecord &retrieveDependencyInfo(const TpiSource *Source) { assert(Source->Kind == TpiSource::UsingPCH); - return ((UsePrecompSource *)Source)->PrecompDependency; + return ((const UsePrecompSource *)Source)->PrecompDependency; } template <> -const TypeServer2Record &retrieveDependencyInfo(TpiSource *Source) { +const TypeServer2Record &retrieveDependencyInfo(const TpiSource *Source) { assert(Source->Kind == TpiSource::UsingPDB); - return ((UseTypeServerSource *)Source)->TypeServerDependency; + return ((const UseTypeServerSource *)Source)->TypeServerDependency; } } // namespace coff } // namespace lld + +std::map> + TypeServerSource::Instances; + +// Make a PDB path assuming the PDB is in the same folder as the OBJ +static std::string getPdbBaseName(const ObjFile *File, StringRef TSPath) { + StringRef LocalPath = + !File->ParentName.empty() ? File->ParentName : File->getName(); + std::string Path = sys::path::parent_path(LocalPath); + + // Currently, type server PDBs are only created by MSVC cl, which only runs + // on Windows, so we can assume type server paths are Windows style. + return Path + sys::path::filename(TSPath, sys::path::Style::windows).str(); +} + +// The casing of the PDB path stamped in the OBJ can differ from the actual path +// on disk. With this, we ensure to always use lowercase as a key for the +// PDBInputFile::Instances map, at least on Windows. +static std::string normalizePdbPath(StringRef path) { +#if defined(_WIN32) + return path.lower(); +#else // LINUX + return path; +#endif +} + +// If existing, return the actual PDB path on disk. +static Optional findPdbPath(StringRef PDBPath, + const ObjFile *DependentFile) { + // Ensure the file exists before anything else. In some cases, if the path + // points to a removable device, Driver::enqueuePath() would fail with an + // error (EAGAIN, "resource unavailable try again") which we want to skip + // silently. + if (llvm::sys::fs::exists(PDBPath)) + return normalizePdbPath(PDBPath); + std::string Ret = getPdbBaseName(DependentFile, PDBPath); + if (llvm::sys::fs::exists(Ret)) + return normalizePdbPath(Ret); + return None; +} + +// Fetch the PDB instance that was already loaded by the COFF Driver. +Expected +TypeServerSource::findFromFile(const ObjFile *DependentFile) { + const TypeServer2Record &TS = + retrieveDependencyInfo(DependentFile->DebugTypesObj); + + Optional P = findPdbPath(TS.Name, DependentFile); + if (!P) + return createFileError(TS.Name, errorCodeToError(std::error_code( + ENOENT, std::generic_category()))); + + auto It = TypeServerSource::Instances.find(*P); + // The PDB file exists on disk, at this point we expect it to have been + // inserted in the map by TypeServerSource::loadPDB() + assert(It != TypeServerSource::Instances.end()); + + std::pair &PDB = It->second; + + if (!PDB.second) + return createFileError( + *P, createStringError(inconvertibleErrorCode(), PDB.first.c_str())); + + pdb::PDBFile &PDBFile = (PDB.second)->Session->getPDBFile(); + pdb::InfoStream &Info = cantFail(PDBFile.getPDBInfoStream()); + + // Just because a file with a matching name was found doesn't mean it can be + // used. The GUID and Age must match between the PDB header and the OBJ + // TypeServer2 record. The 'Age' is used by MSVC incremental compilation. + if (Info.getGuid() != TS.getGuid() || Info.getAge() != TS.getAge()) + return createFileError( + TS.Name, + make_error(pdb::pdb_error_code::signature_out_of_date)); + + return PDB.second; +} + +// FIXME: Temporary interface until PDBLinker::maybeMergeTypeServerPDB() is +// moved here. +Expected +lld::coff::findTypeServerSource(const ObjFile *F) { + Expected TS = TypeServerSource::findFromFile(F); + if (!TS) + return TS.takeError(); + return TS.get()->Session.get(); +} + +// Queue a PDB type server for loading in the COFF Driver +void TypeServerSource::enqueue(const ObjFile *DependentFile, + const TypeServer2Record &TS) { + // Start by finding where the PDB is located (either the record path or next + // to the OBJ file) + Optional P = findPdbPath(TS.Name, DependentFile); + if (!P) + return; + auto It = TypeServerSource::Instances.emplace( + *P, std::pair{}); + if (!It.second) + return; // another OBJ already scheduled this PDB for load + + Driver->enqueuePath(*P, false); +} + +// Create an instance of TypeServerSource or an error string if the PDB couldn't +// be loaded. The error message will be displayed later, when the referring OBJ +// will be merged in. NOTE - a PDB load failure is not a link error: some +// debug info will simply be missing from the final PDB - that is the default +// accepted behavior. +void lld::coff::loadTypeServerSource(llvm::MemoryBufferRef M) { + std::string Path = normalizePdbPath(M.getBufferIdentifier()); + + Expected TS = TypeServerSource::getInstance(M); + if (!TS) + TypeServerSource::Instances[Path] = {toString(TS.takeError()), nullptr}; + else + TypeServerSource::Instances[Path] = {{}, *TS}; +} + +Expected TypeServerSource::getInstance(MemoryBufferRef M) { + std::unique_ptr ISession; + Error Err = pdb::NativeSession::createFromPdb( + MemoryBuffer::getMemBuffer(M, false), ISession); + if (Err) + return std::move(Err); + + std::unique_ptr Session( + static_cast(ISession.release())); + + pdb::PDBFile &PDBFile = Session->getPDBFile(); + Expected Info = PDBFile.getPDBInfoStream(); + // All PDB Files should have an Info stream. + if (!Info) + return Info.takeError(); + return new TypeServerSource(M, Session.release()); +} Index: COFF/Driver.h =================================================================== --- COFF/Driver.h +++ COFF/Driver.h @@ -77,6 +77,8 @@ MemoryBufferRef takeBuffer(std::unique_ptr MB); + void enqueuePath(StringRef Path, bool WholeArchive); + private: std::unique_ptr Tar; // for /linkrepro @@ -120,8 +122,6 @@ void addArchiveBuffer(MemoryBufferRef MBRef, StringRef SymName, StringRef ParentName, uint64_t OffsetInArchive); - void enqueuePath(StringRef Path, bool WholeArchive); - void enqueueTask(std::function Task); bool run(); Index: COFF/Driver.cpp =================================================================== --- COFF/Driver.cpp +++ COFF/Driver.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "DebugTypes.h" #include "Driver.h" #include "Config.h" #include "ICF.h" @@ -181,6 +182,9 @@ case file_magic::coff_import_library: Symtab->addFile(make(MBRef)); break; + case file_magic::pdb: + loadTypeServerSource(MBRef); + break; case file_magic::coff_cl_gl_object: error(Filename + ": is not a native COFF file. Recompile without /GL"); break; Index: COFF/PDB.cpp =================================================================== --- COFF/PDB.cpp +++ COFF/PDB.cpp @@ -175,11 +175,6 @@ llvm::SmallString<128> NativePath; - /// A list of other PDBs which are loaded during the linking process and which - /// we need to keep around since the linking operation may reference pointers - /// inside of these PDBs. - llvm::SmallVector, 2> LoadedPDBs; - std::vector SectionMap; /// Type index mappings of type server PDBs that we've loaded so far. @@ -189,10 +184,6 @@ /// far. std::map PrecompTypeIndexMappings; - /// List of TypeServer PDBs which cannot be loaded. - /// Cached to prevent repeated load attempts. - std::map MissingTypeServerPDBs; - // For statistics uint64_t GlobalSymbols = 0; uint64_t ModuleSymbols = 0; @@ -416,115 +407,26 @@ return *ObjectIndexMap; } -static Expected> -tryToLoadPDB(const codeview::GUID &GuidFromObj, StringRef TSPath) { - // Ensure the file exists before anything else. We want to return ENOENT, - // "file not found", even if the path points to a removable device (in which - // case the return message would be EAGAIN, "resource unavailable try again") - if (!llvm::sys::fs::exists(TSPath)) - return errorCodeToError(std::error_code(ENOENT, std::generic_category())); - - ErrorOr> MBOrErr = MemoryBuffer::getFile( - TSPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false); - if (!MBOrErr) - return errorCodeToError(MBOrErr.getError()); - - std::unique_ptr ThisSession; - if (auto EC = pdb::NativeSession::createFromPdb( - MemoryBuffer::getMemBuffer(Driver->takeBuffer(std::move(*MBOrErr)), - /*RequiresNullTerminator=*/false), - ThisSession)) - return std::move(EC); - - std::unique_ptr NS( - static_cast(ThisSession.release())); - pdb::PDBFile &File = NS->getPDBFile(); - auto ExpectedInfo = File.getPDBInfoStream(); - // All PDB Files should have an Info stream. - if (!ExpectedInfo) - return ExpectedInfo.takeError(); - - // Just because a file with a matching name was found and it was an actual - // PDB file doesn't mean it matches. For it to match the InfoStream's GUID - // must match the GUID specified in the TypeServer2 record. - if (ExpectedInfo->getGuid() != GuidFromObj) - return make_error(pdb::pdb_error_code::signature_out_of_date); - - return std::move(NS); -} - Expected PDBLinker::maybeMergeTypeServerPDB(ObjFile *File) { - const TypeServer2Record &TS = - retrieveDependencyInfo(File->DebugTypesObj); - - const codeview::GUID &TSId = TS.getGuid(); - StringRef TSPath = TS.getName(); - - // First, check if the PDB has previously failed to load. - auto PrevErr = MissingTypeServerPDBs.find(TSId); - if (PrevErr != MissingTypeServerPDBs.end()) - return createFileError( - TSPath, - make_error(PrevErr->second, inconvertibleErrorCode())); - - // Second, check if we already loaded a PDB with this GUID. Return the type - // index mapping if we have it. - auto Insertion = TypeServerIndexMappings.insert({TSId, CVIndexMap()}); - CVIndexMap &IndexMap = Insertion.first->second; - if (!Insertion.second) - return IndexMap; + Expected PDBSession = findTypeServerSource(File); + if (!PDBSession) + return PDBSession.takeError(); + + pdb::PDBFile &PDBFile = PDBSession.get()->getPDBFile(); + pdb::InfoStream &Info = cantFail(PDBFile.getPDBInfoStream()); + + auto It = TypeServerIndexMappings.emplace(Info.getGuid(), CVIndexMap()); + CVIndexMap &IndexMap = It.first->second; + if (!It.second) + return IndexMap; // already merged // Mark this map as a type server map. IndexMap.IsTypeServerMap = true; - // Check for a PDB at: - // 1. The given file path - // 2. Next to the object file or archive file - auto ExpectedSession = handleExpected( - tryToLoadPDB(TSId, TSPath), - [&]() { - StringRef LocalPath = - !File->ParentName.empty() ? File->ParentName : File->getName(); - SmallString<128> Path = sys::path::parent_path(LocalPath); - // Currently, type server PDBs are only created by cl, which only runs - // on Windows, so we can assume type server paths are Windows style. - sys::path::append( - Path, sys::path::filename(TSPath, sys::path::Style::windows)); - return tryToLoadPDB(TSId, Path); - }, - [&](std::unique_ptr EC) -> Error { - auto SysErr = EC->convertToErrorCode(); - // Only re-try loading if the previous error was "No such file or - // directory" - if (SysErr.category() == std::generic_category() && - SysErr.value() == ENOENT) - return Error::success(); - return Error(std::move(EC)); - }); - - if (auto E = ExpectedSession.takeError()) { - TypeServerIndexMappings.erase(TSId); - - // Flatten the error to a string, for later display, if the error occurs - // again on the same PDB. - std::string ErrMsg; - raw_string_ostream S(ErrMsg); - S << E; - MissingTypeServerPDBs.emplace(TSId, S.str()); - - return createFileError(TSPath, std::move(E)); - } - - pdb::NativeSession *Session = ExpectedSession->get(); - - // Keep a strong reference to this PDB, so that it's safe to hold pointers - // into the file. - LoadedPDBs.push_back(std::move(*ExpectedSession)); - - auto ExpectedTpi = Session->getPDBFile().getPDBTpiStream(); + Expected ExpectedTpi = PDBFile.getPDBTpiStream(); if (auto E = ExpectedTpi.takeError()) fatal("Type server does not have TPI stream: " + toString(std::move(E))); - auto ExpectedIpi = Session->getPDBFile().getPDBIpiStream(); + Expected ExpectedIpi = PDBFile.getPDBIpiStream(); if (auto E = ExpectedIpi.takeError()) fatal("Type server does not have TPI stream: " + toString(std::move(E))); Index: test/COFF/Inputs/pdb-type-server-valid-signature.yaml =================================================================== --- test/COFF/Inputs/pdb-type-server-valid-signature.yaml +++ test/COFF/Inputs/pdb-type-server-valid-signature.yaml @@ -69,7 +69,7 @@ - Kind: LF_TYPESERVER2 TypeServer2: Guid: '{8DABD2A0-28FF-CB43-9BAF-175B77B76414}' - Age: 18 + Age: 1 Name: 'pdb-diff-cl.pdb' - Name: '.text$mn' Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] Index: test/COFF/pdb-type-server-invalid-signature.yaml =================================================================== --- test/COFF/pdb-type-server-invalid-signature.yaml +++ test/COFF/pdb-type-server-invalid-signature.yaml @@ -4,7 +4,7 @@ # RUN: lld-link %t.obj -out:%t.exe -debug -pdb:%t.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s # RUN: cd %S -# CHECK: warning: Cannot use debug info for {{.*}}.obj +# CHECK: warning: Cannot use debug info for '{{.*}}.obj' # CHECK-NEXT: The signature does not match; the file(s) might be out of date # Also test a valid match @@ -14,7 +14,7 @@ # RUN: lld-link %t2.obj -out:%t2.exe -debug -pdb:%t2.pdb -nodefaultlib -entry:main 2>&1 | FileCheck %s -check-prefix=VALID-SIGNATURE -allow-empty # RUN: cd %S -# VALID-SIGNATURE-NOT: warning: Cannot use debug info for {{.*}}.obj +# VALID-SIGNATURE-NOT: warning: Cannot use debug info for '{{.*}}.obj' # VALID-SIGNATURE-NOT: The signature does not match; the file(s) might be out of date --- !COFF