Index: include/clang/Basic/FileManager.h =================================================================== --- include/clang/Basic/FileManager.h +++ include/clang/Basic/FileManager.h @@ -60,6 +60,7 @@ bool IsNamedPipe; bool InPCH; bool IsValid; // Is this \c FileEntry initialized and valid? + bool IsVirtual; // Is this \c FileEntry a virtual file? /// \brief The open file, if it is owned by the \p FileEntry. mutable std::unique_ptr File; @@ -69,7 +70,8 @@ public: FileEntry() - : UniqueID(0, 0), IsNamedPipe(false), InPCH(false), IsValid(false) + : UniqueID(0, 0), IsNamedPipe(false), InPCH(false), IsValid(false), + IsVirtual(false) {} // FIXME: this is here to allow putting FileEntry in std::map. Once we have @@ -77,12 +79,14 @@ /// Intentionally does not copy fields that are not set in an uninitialized /// \c FileEntry. FileEntry(const FileEntry &FE) : UniqueID(FE.UniqueID), - IsNamedPipe(FE.IsNamedPipe), InPCH(FE.InPCH), IsValid(FE.IsValid) { + IsNamedPipe(FE.IsNamedPipe), InPCH(FE.InPCH), IsValid(FE.IsValid), + IsVirtual(FE.IsVirtual) { assert(!isValid() && "Cannot copy an initialized FileEntry"); } const char *getName() const { return Name; } bool isValid() const { return IsValid; } + bool isVirtual() const { return IsVirtual; } off_t getSize() const { return Size; } unsigned getUID() const { return UID; } const llvm::sys::fs::UniqueID &getUniqueID() const { return UniqueID; } Index: include/clang/Frontend/ASTUnit.h =================================================================== --- include/clang/Frontend/ASTUnit.h +++ include/clang/Frontend/ASTUnit.h @@ -425,7 +425,8 @@ void CleanTemporaryFiles(); bool Parse(std::shared_ptr PCHContainerOps, - std::unique_ptr OverrideMainBuffer); + std::unique_ptr OverrideMainBuffer, + std::unique_ptr PCHFileMgr = nullptr); struct ComputedPreamble { llvm::MemoryBuffer *Buffer; @@ -446,8 +447,9 @@ std::unique_ptr getMainBufferWithPrecompiledPreamble( std::shared_ptr PCHContainerOps, - const CompilerInvocation &PreambleInvocationIn, bool AllowRebuild = true, - unsigned MaxLines = 0); + const CompilerInvocation &PreambleInvocationIn, + std::unique_ptr &PCHFileMgr, + bool AllowRebuild = true, unsigned MaxLines = 0); void RealizeTopLevelDeclsFromPreamble(); /// \brief Transfers ownership of the objects (like SourceManager) from @@ -860,7 +862,8 @@ /// \returns True if a failure occurred that causes the ASTUnit not to /// contain any translation-unit information, false otherwise. bool Reparse(std::shared_ptr PCHContainerOps, - ArrayRef RemappedFiles = None); + ArrayRef RemappedFiles = None, + FileManager *FileMgr = nullptr); /// \brief Perform code completion at the given file, line, and /// column within this translation unit. Index: lib/Basic/FileManager.cpp =================================================================== --- lib/Basic/FileManager.cpp +++ lib/Basic/FileManager.cpp @@ -301,6 +301,11 @@ return &UFE; } + if (UFE.isVirtual()) { + UFE.Name = InterndFileName; + return &UFE; + } + // Otherwise, we don't have this file yet, add it. UFE.Name = InterndFileName; UFE.Size = Data.Size; @@ -381,6 +386,7 @@ UFE->Dir = DirInfo; UFE->UID = NextFileUID++; UFE->File.reset(); + UFE->IsVirtual = true; return UFE; } Index: lib/Frontend/ASTUnit.cpp =================================================================== --- lib/Frontend/ASTUnit.cpp +++ lib/Frontend/ASTUnit.cpp @@ -1032,7 +1032,8 @@ /// \returns True if a failure occurred that causes the ASTUnit not to /// contain any translation-unit information, false otherwise. bool ASTUnit::Parse(std::shared_ptr PCHContainerOps, - std::unique_ptr OverrideMainBuffer) { + std::unique_ptr OverrideMainBuffer, + std::unique_ptr PCHFileMgr) { SavedMainFileBuffer.reset(); if (!Invocation) @@ -1078,6 +1079,8 @@ // Configure the various subsystems. LangOpts = Clang->getInvocation().LangOpts; FileSystemOpts = Clang->getFileSystemOpts(); + if (PCHFileMgr) + FileMgr = PCHFileMgr.release(); if (!FileMgr) { Clang->createFileManager(); FileMgr = &Clang->getFileManager(); @@ -1316,6 +1319,23 @@ return OutDiag; } +static IntrusiveRefCntPtr createPreambleVFSOverlay( + IntrusiveRefCntPtr RawPreambleVFS, + IntrusiveRefCntPtr CustomVFS) { + IntrusiveRefCntPtr + Overlay(new vfs::OverlayFileSystem(RawPreambleVFS)); + llvm::ErrorOr PrevWorkingDir + = CustomVFS->getCurrentWorkingDirectory(); + Overlay->pushOverlay(CustomVFS); + // pushOverlay changes the working directory, but we can't assume that the + // two filesystems are compatible that way. We only underlay the other + // filesystem so that the PCH file can be written/read from it anyway, + // and for that we use an absolute path in the first place. + if (PrevWorkingDir) + CustomVFS->setCurrentWorkingDirectory(PrevWorkingDir.get()); + return Overlay; +} + /// \brief Attempt to build or re-use a precompiled preamble when (re-)parsing /// the source file. /// @@ -1339,8 +1359,9 @@ std::unique_ptr ASTUnit::getMainBufferWithPrecompiledPreamble( std::shared_ptr PCHContainerOps, - const CompilerInvocation &PreambleInvocationIn, bool AllowRebuild, - unsigned MaxLines) { + const CompilerInvocation &PreambleInvocationIn, + std::unique_ptr &PCHFileMgr, + bool AllowRebuild, unsigned MaxLines) { IntrusiveRefCntPtr PreambleInvocation(new CompilerInvocation(PreambleInvocationIn)); @@ -1448,6 +1469,11 @@ PreambleInvocation->getDiagnosticOpts()); getDiagnostics().setNumWarnings(NumWarningsInPreamble); + IntrusiveRefCntPtr VFS = vfs::getRealFileSystem(); + if (VFS && FileMgr && FileMgr->getVirtualFileSystem() != VFS) + PCHFileMgr.reset(new FileManager(FileSystemOpts, + createPreambleVFSOverlay(VFS, FileMgr->getVirtualFileSystem()))); + return llvm::MemoryBuffer::getMemBufferCopy( NewPreamble.Buffer->getBuffer(), FrontendOpts.Inputs[0].getFile()); } @@ -1558,12 +1584,14 @@ TopLevelDeclsInPreamble.clear(); PreambleDiagnostics.clear(); - IntrusiveRefCntPtr VFS = - createVFSFromCompilerInvocation(Clang->getInvocation(), getDiagnostics()); + // Create a file manager object to provide access to and cache the filesystem. + IntrusiveRefCntPtr VFS + = createVFSFromCompilerInvocation(Clang->getInvocation(), getDiagnostics()); if (!VFS) return nullptr; - - // Create a file manager object to provide access to and cache the filesystem. + if (FileMgr && FileMgr->getVirtualFileSystem() != VFS) + VFS = createPreambleVFSOverlay(VFS, FileMgr->getVirtualFileSystem()); + PCHFileMgr.reset(new FileManager(Clang->getFileSystemOpts(), VFS)); Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS)); // Create the source manager. @@ -1883,10 +1911,12 @@ ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts()); std::unique_ptr OverrideMainBuffer; + std::unique_ptr PCHFileMgr; if (PrecompilePreambleAfterNParses > 0) { PreambleRebuildCounter = PrecompilePreambleAfterNParses; OverrideMainBuffer = - getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation); + getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, + PCHFileMgr); } SimpleTimer ParsingTimer(WantTiming); @@ -1896,7 +1926,8 @@ llvm::CrashRecoveryContextCleanupRegistrar MemBufferCleanup(OverrideMainBuffer.get()); - return Parse(PCHContainerOps, std::move(OverrideMainBuffer)); + return Parse(PCHContainerOps, std::move(OverrideMainBuffer), + std::move(PCHFileMgr)); } std::unique_ptr ASTUnit::LoadFromCompilerInvocation( @@ -2027,7 +2058,8 @@ } bool ASTUnit::Reparse(std::shared_ptr PCHContainerOps, - ArrayRef RemappedFiles) { + ArrayRef RemappedFiles, + FileManager *FileMgr) { if (!Invocation) return true; @@ -2049,13 +2081,18 @@ // If we have a preamble file lying around, or if we might try to // build a precompiled preamble, do so now. + if (FileMgr) + this->FileMgr = FileMgr; std::unique_ptr OverrideMainBuffer; + std::unique_ptr PCHFileMgr; if (!getPreambleFile(this).empty() || PreambleRebuildCounter > 0) OverrideMainBuffer = - getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation); + getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, + PCHFileMgr); + if (PCHFileMgr) + this->FileMgr = PCHFileMgr.release(); // Clear out the diagnostics state. - FileMgr.reset(); getDiagnostics().Reset(); ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts()); if (OverrideMainBuffer) @@ -2424,10 +2461,11 @@ if (!llvm::sys::fs::getUniqueID(CompleteFilePath, CompleteFileID)) { std::string MainPath(OriginalSourceFile); llvm::sys::fs::UniqueID MainID; + std::unique_ptr PCHFileMgr; if (!llvm::sys::fs::getUniqueID(MainPath, MainID)) { if (CompleteFileID == MainID && Line > 1) OverrideMainBuffer = getMainBufferWithPrecompiledPreamble( - PCHContainerOps, *CCInvocation, false, Line - 1); + PCHContainerOps, *CCInvocation, PCHFileMgr, false, Line - 1); } } } Index: unittests/Frontend/CMakeLists.txt =================================================================== --- unittests/Frontend/CMakeLists.txt +++ unittests/Frontend/CMakeLists.txt @@ -5,6 +5,7 @@ add_clang_unittest(FrontendTests FrontendActionTest.cpp CodeGenActionTest.cpp + PchPreambleTest.cpp ) target_link_libraries(FrontendTests clangAST Index: unittests/Frontend/PchPreambleTest.cpp =================================================================== --- /dev/null +++ unittests/Frontend/PchPreambleTest.cpp @@ -0,0 +1,155 @@ +//====-- unittests/Frontend/PchPreambleTest.cpp - FrontendAction tests ---====// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Frontend/FrontendOptions.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/FileManager.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "gtest/gtest.h" +#include + +using namespace llvm; +using namespace clang; + +namespace { + +class ReadCountingInMemoryFileSystem : public vfs::InMemoryFileSystem +{ + std::map ReadCounts; + +public: + ErrorOr> openFileForRead(const Twine &Path) override + { + SmallVector PathVec; + Path.toVector(PathVec); + llvm::sys::path::remove_dots(PathVec, true); + ++ReadCounts[std::string(PathVec.begin(), PathVec.end())]; + return InMemoryFileSystem::openFileForRead(Path); + } + + unsigned GetReadCount(const Twine &Path) const + { + auto it = ReadCounts.find(Path.str()); + return it == ReadCounts.end() ? 0 : it->second; + } +}; + +class PchPreambleTest : public ::testing::Test { + IntrusiveRefCntPtr VFS; + StringMap RemappedFiles; + std::shared_ptr PCHContainerOpts; + FileSystemOptions FSOpts; + +public: + void SetUp() override { + VFS = new ReadCountingInMemoryFileSystem(); + // We need the working directory to be set to something absolute, + // otherwise it ends up being inadvertently set to the current + // working directory in the real file system due to a series of + // unfortunate conditions interacting badly. + // What's more, this path *must* be absolute on all (real) + // filesystems, so just '/' won't work (e.g. on Win32). + VFS->setCurrentWorkingDirectory("//./"); + } + + void TearDown() override { + } + + void AddFile(const std::string &Filename, const std::string &Contents) { + ::time_t now; + ::time(&now); + VFS->addFile(Filename, now, MemoryBuffer::getMemBufferCopy(Contents, Filename)); + } + + void RemapFile(const std::string &Filename, const std::string &Contents) { + RemappedFiles[Filename] = Contents; + } + + std::unique_ptr ParseAST(const std::string &EntryFile) { + PCHContainerOpts = std::make_shared(); + CompilerInvocation *CI = new CompilerInvocation; + CI->getFrontendOpts().Inputs.push_back( + FrontendInputFile(EntryFile, FrontendOptions::getInputKindForExtension( + llvm::sys::path::extension(EntryFile).substr(1)))); + + CI->getTargetOpts().Triple = "i386-unknown-linux-gnu"; + + CI->getPreprocessorOpts().RemappedFileBuffers = GetRemappedFiles(); + + PreprocessorOptions &PPOpts = CI->getPreprocessorOpts(); + PPOpts.RemappedFilesKeepOriginalName = true; + + IntrusiveRefCntPtr + Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions)); + + FileManager *FileMgr = new FileManager(FSOpts, VFS); + + std::unique_ptr AST = ASTUnit::LoadFromCompilerInvocation( + CI, PCHContainerOpts, Diags, FileMgr, false, false, + /*PrecompilePreambleAfterNParses=*/1); + return AST; + } + + bool ReparseAST(const std::unique_ptr &AST) { + FileManager *FileMgr = new FileManager(FSOpts, VFS); + bool reparseFailed = AST->Reparse(PCHContainerOpts, GetRemappedFiles(), FileMgr); + return reparseFailed; + } + + unsigned GetFileReadCount(const std::string &Filename) const { + return VFS->GetReadCount(Filename); + } + +private: + std::vector> + GetRemappedFiles() const { + std::vector> Remapped; + for (const auto &RemappedFile : RemappedFiles) { + std::unique_ptr buf = MemoryBuffer::getMemBufferCopy( + RemappedFile.second, RemappedFile.first()); + Remapped.emplace_back(RemappedFile.first(), buf.release()); + } + return Remapped; + } +}; + +TEST_F(PchPreambleTest, ReparseWithOverriddenFileDoesNotInvalidatePreamble) { + std::string Header1 = "//./header1.h"; + std::string Header2 = "//./header2.h"; + std::string MainName = "//./main.cpp"; + AddFile(Header1, ""); + AddFile(Header2, "#pragma once"); + AddFile(MainName, + "#include \"//./foo/../header1.h\"\n" + "#include \"//./foo/../header2.h\"\n" + "int main() { return foo(); }"); + RemapFile(Header1, "inline int foo2() { return 0; }"); + + std::unique_ptr AST(ParseAST(MainName)); + ASSERT_TRUE(AST.get()); + + unsigned initialCounts[] = { + GetFileReadCount(MainName), + GetFileReadCount(Header1), + GetFileReadCount(Header2) + }; + + ASSERT_FALSE(ReparseAST(AST)); + + ASSERT_NE(initialCounts[0], GetFileReadCount(MainName)); + ASSERT_EQ(initialCounts[1], GetFileReadCount(Header1)); + ASSERT_EQ(initialCounts[2], GetFileReadCount(Header2)); +} + +} // anonymous namespace