Index: clang/include/clang/Frontend/ASTUnit.h =================================================================== --- clang/include/clang/Frontend/ASTUnit.h +++ clang/include/clang/Frontend/ASTUnit.h @@ -53,6 +53,7 @@ namespace vfs { class FileSystem; +class InMemoryFileSystem; } // namespace vfs } // namespace llvm @@ -108,6 +109,16 @@ private: std::shared_ptr LangOpts; IntrusiveRefCntPtr Diagnostics; + + /// The top-level filesystem. + llvm::IntrusiveRefCntPtr FS; + + /// Filesystem for remapped file buffers. + llvm::IntrusiveRefCntPtr RemappedFilesFS; + + /// The base filesystem from the CompilerInvocation. + llvm::IntrusiveRefCntPtr BaseFS; + IntrusiveRefCntPtr FileMgr; IntrusiveRefCntPtr SourceMgr; IntrusiveRefCntPtr ModuleCache; @@ -368,13 +379,11 @@ explicit ASTUnit(bool MainFileIsAST); bool Parse(std::shared_ptr PCHContainerOps, - std::unique_ptr OverrideMainBuffer, - IntrusiveRefCntPtr VFS); + std::unique_ptr OverrideMainBuffer); std::unique_ptr getMainBufferWithPrecompiledPreamble( std::shared_ptr PCHContainerOps, - CompilerInvocation &PreambleInvocationIn, - IntrusiveRefCntPtr VFS, bool AllowRebuild = true, + CompilerInvocation &PreambleInvocationIn, bool AllowRebuild = true, unsigned MaxLines = 0); void RealizeTopLevelDeclsFromPreamble(); @@ -659,7 +668,8 @@ /// A mapping from a file name to the memory buffer that stores the /// remapped contents of that file. - using RemappedFile = std::pair; + using RemappedFile = + std::pair>; /// Create a ASTUnit. Gets ownership of the passed CompilerInvocation. static std::unique_ptr @@ -706,17 +716,11 @@ /// of this translation unit should be precompiled, to improve the performance /// of reparsing. Set to zero to disable preambles. /// - /// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses. - /// Note that preamble is saved to a temporary directory on a RealFileSystem, - /// so in order for it to be loaded correctly, VFS should have access to - /// it(i.e., be an overlay over RealFileSystem). - /// /// \returns \c true if a catastrophic failure occurred (which means that the /// \c ASTUnit itself is invalid), or \c false otherwise. bool LoadFromCompilerInvocation( std::shared_ptr PCHContainerOps, - unsigned PrecompilePreambleAfterNParses, - IntrusiveRefCntPtr VFS); + unsigned PrecompilePreambleAfterNParses); public: /// Create an ASTUnit from a source file, via a CompilerInvocation @@ -779,7 +783,8 @@ std::shared_ptr PCHContainerOps, IntrusiveRefCntPtr Diags, FileManager *FileMgr, unsigned PrecompilePreambleAfterNParses = 0, - bool UserFilesAreVolatile = false); + bool UserFilesAreVolatile = false, + Optional> RemappedFiles = None); /// LoadFromCommandLine - Create an ASTUnit from a vector of command line /// arguments, which must specify exactly one source file. @@ -802,12 +807,6 @@ /// (e.g. because the PCH could not be loaded), this accepts the ASTUnit /// mainly to allow the caller to see the diagnostics. /// - /// \param VFS - A llvm::vfs::FileSystem to be used for all file accesses. - /// Note that preamble is saved to a temporary directory on a RealFileSystem, - /// so in order for it to be loaded correctly, VFS should have access to - /// it(i.e., be an overlay over RealFileSystem). RealFileSystem will be used - /// if \p VFS is nullptr. - /// // FIXME: Move OnlyLocalDecls, UseBumpAllocator to setters on the ASTUnit, we // shouldn't need to specify them at construction time. static ASTUnit *LoadFromCommandLine( @@ -816,7 +815,7 @@ IntrusiveRefCntPtr Diags, StringRef ResourceFilesPath, bool OnlyLocalDecls = false, CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None, - ArrayRef RemappedFiles = None, + Optional> RemappedFiles = None, unsigned PrecompilePreambleAfterNParses = 0, TranslationUnitKind TUKind = TU_Complete, bool CacheCodeCompletionResults = false, @@ -842,7 +841,7 @@ /// \returns True if a failure occurred that causes the ASTUnit not to /// contain any translation-unit information, false otherwise. bool Reparse(std::shared_ptr PCHContainerOps, - ArrayRef RemappedFiles = None, + Optional> RemappedFiles = None, IntrusiveRefCntPtr VFS = nullptr); /// Free data that will be re-generated on the next parse. @@ -871,9 +870,9 @@ /// FIXME: The Diag, LangOpts, SourceMgr, FileMgr, StoredDiagnostics, and /// OwnedBuffers parameters are all disgusting hacks. They will go away. void CodeComplete(StringRef File, unsigned Line, unsigned Column, - ArrayRef RemappedFiles, bool IncludeMacros, - bool IncludeCodePatterns, bool IncludeBriefComments, - CodeCompleteConsumer &Consumer, + Optional> RemappedFiles, + bool IncludeMacros, bool IncludeCodePatterns, + bool IncludeBriefComments, CodeCompleteConsumer &Consumer, std::shared_ptr PCHContainerOps, DiagnosticsEngine &Diag, LangOptions &LangOpts, SmallVectorImpl &StoredDiagnostics, @@ -889,6 +888,18 @@ /// /// \returns True if an error occurred, false otherwise. bool serialize(raw_ostream &OS); + +private: + IntrusiveRefCntPtr computeBaseFS( + const CompilerInvocation &CI, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr MaybeFS = nullptr) const; + void initializeBaseFS(IntrusiveRefCntPtr BaseFS); + void computeAndInitializeFS( + const CompilerInvocation &CI, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr MaybeFS = nullptr); + void reinitializeBaseFS( + IntrusiveRefCntPtr MaybeFS = nullptr); + void remapFiles(Optional> RemappedFiles); }; } // namespace clang Index: clang/include/clang/Frontend/PrecompiledPreamble.h =================================================================== --- clang/include/clang/Frontend/PrecompiledPreamble.h +++ clang/include/clang/Frontend/PrecompiledPreamble.h @@ -106,7 +106,8 @@ /// MainFileBuffer) of the main file. bool CanReuse(const CompilerInvocation &Invocation, const llvm::MemoryBufferRef &MainFileBuffer, - PreambleBounds Bounds, llvm::vfs::FileSystem &VFS) const; + PreambleBounds Bounds, llvm::vfs::FileSystem &VFS, + llvm::vfs::FileSystem *OverriddenFilesFS = nullptr) const; /// Changes options inside \p CI to use PCH from this preamble. Also remaps /// main file to \p MainFileBuffer and updates \p VFS to ensure the preamble Index: clang/lib/Frontend/ASTUnit.cpp =================================================================== --- clang/lib/Frontend/ASTUnit.cpp +++ clang/lib/Frontend/ASTUnit.cpp @@ -775,9 +775,8 @@ AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; AST->Diagnostics = Diags; - IntrusiveRefCntPtr VFS = - llvm::vfs::getRealFileSystem(); - AST->FileMgr = new FileManager(FileSystemOpts, VFS); + AST->initializeBaseFS(llvm::vfs::getRealFileSystem()); + AST->FileMgr = new FileManager(FileSystemOpts, AST->FS); AST->UserFilesAreVolatile = UserFilesAreVolatile; AST->SourceMgr = new SourceManager(AST->getDiagnostics(), AST->getFileManager(), @@ -1094,21 +1093,21 @@ /// \returns True if a failure occurred that causes the ASTUnit not to /// contain any translation-unit information, false otherwise. bool ASTUnit::Parse(std::shared_ptr PCHContainerOps, - std::unique_ptr OverrideMainBuffer, - IntrusiveRefCntPtr VFS) { + std::unique_ptr OverrideMainBuffer) { if (!Invocation) return true; - if (VFS && FileMgr) - assert(VFS == &FileMgr->getVirtualFileSystem() && - "VFS passed to Parse and VFS in FileMgr are different"); + assert((!FileMgr || &FileMgr->getVirtualFileSystem() == FS) && + "ASTUnit has a FileManager without initializing FS"); auto CCInvocation = std::make_shared(*Invocation); if (OverrideMainBuffer) { assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null"); - Preamble->AddImplicitPreamble(*CCInvocation, VFS, OverrideMainBuffer.get()); + Preamble->AddImplicitPreamble(*CCInvocation, FS, OverrideMainBuffer.get()); // VFS may have changed... + if (FileMgr && &FileMgr->getVirtualFileSystem() != FS) + FileMgr->setVirtualFileSystem(FS); } // Create the compiler instance to use for building the AST. @@ -1131,10 +1130,10 @@ // Ensure that Clang has a FileManager with the right VFS, which may have // changed above in AddImplicitPreamble. If VFS is nullptr, rely on // createFileManager to create one. - if (VFS && FileMgr && &FileMgr->getVirtualFileSystem() == VFS) + if (FileMgr && &FileMgr->getVirtualFileSystem() == FS) Clang->setFileManager(&*FileMgr); else - FileMgr = Clang->createFileManager(std::move(VFS)); + FileMgr = Clang->createFileManager(FS); // Recover resources if we crash before exiting this method. llvm::CrashRecoveryContextCleanupRegistrar @@ -1299,13 +1298,12 @@ std::unique_ptr ASTUnit::getMainBufferWithPrecompiledPreamble( std::shared_ptr PCHContainerOps, - CompilerInvocation &PreambleInvocationIn, - IntrusiveRefCntPtr VFS, bool AllowRebuild, + CompilerInvocation &PreambleInvocationIn, bool AllowRebuild, unsigned MaxLines) { auto MainFilePath = PreambleInvocationIn.getFrontendOpts().Inputs[0].getFile(); std::unique_ptr MainFileBuffer = - getBufferForFileHandlingRemapping(PreambleInvocationIn, VFS.get(), + getBufferForFileHandlingRemapping(PreambleInvocationIn, FS.get(), MainFilePath, UserFilesAreVolatile); if (!MainFileBuffer) return nullptr; @@ -1317,7 +1315,7 @@ if (Preamble) { if (Preamble->CanReuse(PreambleInvocationIn, *MainFileBuffer, Bounds, - *VFS)) { + *FS, &*RemappedFilesFS)) { // Okay! We can re-use the precompiled preamble. // Set the state of the diagnostic object to mimic its state @@ -1373,8 +1371,8 @@ PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies = true; llvm::ErrorOr NewPreamble = PrecompiledPreamble::Build( - PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, VFS, - PCHContainerOps, /*StoreInMemory=*/false, Callbacks); + PreambleInvocationIn, MainFileBuffer.get(), Bounds, *Diagnostics, + FS.get(), PCHContainerOps, /*StoreInMemory=*/false, Callbacks); PreambleInvocationIn.getFrontendOpts().SkipFunctionBodies = PreviousSkipFunctionBodies; @@ -1492,12 +1490,11 @@ bool UserFilesAreVolatile) { std::unique_ptr AST(new ASTUnit(false)); ConfigureDiags(Diags, *AST, CaptureDiagnostics); - IntrusiveRefCntPtr VFS = - createVFSFromCompilerInvocation(*CI, *Diags); AST->Diagnostics = Diags; AST->FileSystemOpts = CI->getFileSystemOpts(); AST->Invocation = std::move(CI); - AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS); + AST->computeAndInitializeFS(*AST->Invocation, *Diags); + AST->FileMgr = new FileManager(AST->FileSystemOpts, AST->FS); AST->UserFilesAreVolatile = UserFilesAreVolatile; AST->SourceMgr = new SourceManager(AST->getDiagnostics(), *AST->FileMgr, UserFilesAreVolatile); @@ -1653,13 +1650,10 @@ bool ASTUnit::LoadFromCompilerInvocation( std::shared_ptr PCHContainerOps, - unsigned PrecompilePreambleAfterNParses, - IntrusiveRefCntPtr VFS) { + unsigned PrecompilePreambleAfterNParses) { if (!Invocation) return true; - assert(VFS && "VFS is null"); - // We'll manage file buffers ourselves. Invocation->getPreprocessorOpts().RetainRemappedFileBuffers = true; Invocation->getFrontendOpts().DisableFree = false; @@ -1670,7 +1664,7 @@ if (PrecompilePreambleAfterNParses > 0) { PreambleRebuildCountdown = PrecompilePreambleAfterNParses; OverrideMainBuffer = - getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS); + getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation); getDiagnostics().Reset(); ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts()); } @@ -1682,15 +1676,15 @@ llvm::CrashRecoveryContextCleanupRegistrar MemBufferCleanup(OverrideMainBuffer.get()); - return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS); + return Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer)); } std::unique_ptr ASTUnit::LoadFromCompilerInvocation( std::shared_ptr CI, std::shared_ptr PCHContainerOps, IntrusiveRefCntPtr Diags, FileManager *FileMgr, - unsigned PrecompilePreambleAfterNParses, - bool UserFilesAreVolatile) { + unsigned PrecompilePreambleAfterNParses, bool UserFilesAreVolatile, + Optional> RemappedFiles) { // Create the AST unit. std::unique_ptr AST(new ASTUnit(false)); ConfigureDiags(Diags, *AST, CaptureDiagsKind::None); @@ -1705,6 +1699,12 @@ AST->FileMgr = FileMgr; AST->UserFilesAreVolatile = UserFilesAreVolatile; + // If we have a file manager, this will update its VFS. + AST->computeAndInitializeFS(*AST->Invocation, *Diags); + + // Override any files that need remapping. + AST->remapFiles(std::move(RemappedFiles)); + // Recover resources if we crash before exiting this method. llvm::CrashRecoveryContextCleanupRegistrar ASTUnitCleanup(AST.get()); @@ -1713,8 +1713,7 @@ DiagCleanup(Diags.get()); if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps), - PrecompilePreambleAfterNParses, - &AST->FileMgr->getVirtualFileSystem())) + PrecompilePreambleAfterNParses)) return nullptr; return AST; } @@ -1724,7 +1723,7 @@ std::shared_ptr PCHContainerOps, IntrusiveRefCntPtr Diags, StringRef ResourceFilesPath, bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics, - ArrayRef RemappedFiles, + Optional> RemappedFiles, unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind, bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion, bool AllowPCHWithCompilerErrors, SkipFunctionBodiesScope SkipFunctionBodies, @@ -1747,11 +1746,6 @@ return nullptr; } - // Override any files that need remapping - for (const auto &RemappedFile : RemappedFiles) { - CI->getPreprocessorOpts().addRemappedFile(RemappedFile.first, - RemappedFile.second); - } PreprocessorOptions &PPOpts = CI->getPreprocessorOpts(); PPOpts.RemappedFilesKeepOriginalName = true; PPOpts.AllowPCHWithCompilerErrors = AllowPCHWithCompilerErrors; @@ -1771,15 +1765,17 @@ // Create the AST unit. std::unique_ptr AST; AST.reset(new ASTUnit(false)); + + // Override any files that need remapping + AST->initializeBaseFS(createVFSFromCompilerInvocation(*CI, *Diags)); + AST->remapFiles(std::move(RemappedFiles)); + AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size(); AST->StoredDiagnostics.swap(StoredDiagnostics); ConfigureDiags(Diags, *AST, CaptureDiagnostics); AST->Diagnostics = Diags; AST->FileSystemOpts = CI->getFileSystemOpts(); - IntrusiveRefCntPtr VFS = - llvm::vfs::getRealFileSystem(); - VFS = createVFSFromCompilerInvocation(*CI, *Diags, VFS); - AST->FileMgr = new FileManager(AST->FileSystemOpts, VFS); + AST->FileMgr = new FileManager(AST->FileSystemOpts, AST->FS); AST->ModuleCache = new InMemoryModuleCache; AST->OnlyLocalDecls = OnlyLocalDecls; AST->CaptureDiagnostics = CaptureDiagnostics; @@ -1801,8 +1797,7 @@ ASTUnitCleanup(AST.get()); if (AST->LoadFromCompilerInvocation(std::move(PCHContainerOps), - PrecompilePreambleAfterNParses, - VFS)) { + PrecompilePreambleAfterNParses)) { // Some error occurred, if caller wants to examine diagnostics, pass it the // ASTUnit. if (ErrAST) { @@ -1816,38 +1811,33 @@ } bool ASTUnit::Reparse(std::shared_ptr PCHContainerOps, - ArrayRef RemappedFiles, + Optional> RemappedFiles, IntrusiveRefCntPtr VFS) { if (!Invocation) return true; - if (!VFS) { - assert(FileMgr && "FileMgr is null on Reparse call"); - VFS = &FileMgr->getVirtualFileSystem(); - } - clearFileLevelDecls(); SimpleTimer ParsingTimer(WantTiming); ParsingTimer.setOutput("Reparsing " + getMainFileName()); // Remap files. + reinitializeBaseFS(std::move(VFS)); + remapFiles(std::move(RemappedFiles)); + + // Clear out buffers in the Invocation. PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts(); for (const auto &RB : PPOpts.RemappedFileBuffers) delete RB.second; Invocation->getPreprocessorOpts().clearRemappedFiles(); - for (const auto &RemappedFile : RemappedFiles) { - Invocation->getPreprocessorOpts().addRemappedFile(RemappedFile.first, - RemappedFile.second); - } // If we have a preamble file lying around, or if we might try to // build a precompiled preamble, do so now. std::unique_ptr OverrideMainBuffer; if (Preamble || PreambleRebuildCountdown > 0) OverrideMainBuffer = - getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation, VFS); + getMainBufferWithPrecompiledPreamble(PCHContainerOps, *Invocation); // Clear out the diagnostics state. FileMgr.reset(); @@ -1858,7 +1848,7 @@ // Parse the sources bool Result = - Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer), VFS); + Parse(std::move(PCHContainerOps), std::move(OverrideMainBuffer)); // If we're caching global code-completion results, and the top-level // declarations have changed, clear out the code-completion cache. @@ -2128,7 +2118,7 @@ void ASTUnit::CodeComplete( StringRef File, unsigned Line, unsigned Column, - ArrayRef RemappedFiles, bool IncludeMacros, + Optional> RemappedFiles, bool IncludeMacros, bool IncludeCodePatterns, bool IncludeBriefComments, CodeCompleteConsumer &Consumer, std::shared_ptr PCHContainerOps, @@ -2211,20 +2201,21 @@ Language::LLVM_IR && "IR inputs not support here!"); + // Remap files. + reinitializeBaseFS(); + remapFiles(std::move(RemappedFiles)); + // Initialize file and source managers up front so that the caller can access // them after. Clang->createFileManager(FS); + Clang->createSourceManager(Clang->getFileManager()); FileMgr = &Clang->getFileManager(); - Clang->createSourceManager(*FileMgr); SourceMgr = &Clang->getSourceManager(); - // Remap files. - PreprocessorOpts.clearRemappedFiles(); + // Clear out buffers in the invocation and prepare for any overridden buffers + // there by requesting they be retained. PreprocessorOpts.RetainRemappedFileBuffers = true; - for (const auto &RemappedFile : RemappedFiles) { - PreprocessorOpts.addRemappedFile(RemappedFile.first, RemappedFile.second); - OwnedBuffers.push_back(RemappedFile.second); - } + PreprocessorOpts.clearRemappedFiles(); // Use the code completion consumer we were given, but adding any cached // code-completion results. @@ -2233,8 +2224,8 @@ Clang->setCodeCompletionConsumer(AugmentedConsumer); auto getUniqueID = - [&FileMgr](StringRef Filename) -> Optional { - if (auto Status = FileMgr.getVirtualFileSystem().status(Filename)) + [this](StringRef Filename) -> Optional { + if (auto Status = BaseFS->status(Filename)) return Status->getUniqueID(); return None; }; @@ -2255,7 +2246,7 @@ std::unique_ptr OverrideMainBuffer; if (Preamble && Line > 1 && hasSameUniqueID(File, OriginalSourceFile)) { OverrideMainBuffer = getMainBufferWithPrecompiledPreamble( - PCHContainerOps, Inv, &FileMgr.getVirtualFileSystem(), false, Line - 1); + PCHContainerOps, Inv, false, Line - 1); } // If the main file has been overridden due to the use of a preamble, @@ -2264,9 +2255,7 @@ assert(Preamble && "No preamble was built, but OverrideMainBuffer is not null"); - IntrusiveRefCntPtr VFS = - &FileMgr.getVirtualFileSystem(); - Preamble->AddImplicitPreamble(Clang->getInvocation(), VFS, + Preamble->AddImplicitPreamble(Clang->getInvocation(), FS, OverrideMainBuffer.get()); // FIXME: there is no way to update VFS if it was changed by // AddImplicitPreamble as FileMgr is accepted as a parameter by this method. @@ -2288,6 +2277,9 @@ if (llvm::Error Err = Act->Execute()) { consumeError(std::move(Err)); // FIXME this drops errors on the floor. } + + transferASTDataFromCompilerInstance(*Clang); + Act->EndSourceFile(); } } @@ -2711,3 +2703,64 @@ void ASTUnit::ConcurrencyState::finish() {} #endif // NDEBUG + +void ASTUnit::initializeBaseFS(IntrusiveRefCntPtr BaseFS) { + assert(BaseFS && "Given empty base filesystem?"); + if (this->BaseFS == BaseFS || this->FS == BaseFS) + return; + + llvm::IntrusiveRefCntPtr OverlayFS( + new llvm::vfs::OverlayFileSystem(BaseFS)); + RemappedFilesFS = new llvm::vfs::InMemoryFileSystem; + OverlayFS->pushOverlay(RemappedFilesFS); + FS = std::move(OverlayFS); + this->BaseFS = std::move(BaseFS); + if (FileMgr) + FileMgr->setVirtualFileSystem(FS); +} + +void ASTUnit::reinitializeBaseFS( + IntrusiveRefCntPtr MaybeFS) { + assert(FS && "Resetting without initializing?"); + assert(BaseFS && "Resetting without initializing?"); + assert(RemappedFilesFS && "Resetting without initializing?"); + if (!MaybeFS) + MaybeFS = std::move(BaseFS); + else + BaseFS.reset(); + + RemappedFilesFS.reset(); + FS.reset(); + + // Reinitialize. + initializeBaseFS(std::move(MaybeFS)); +} + +IntrusiveRefCntPtr ASTUnit::computeBaseFS( + const CompilerInvocation &CI, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr MaybeFS) const { + if (MaybeFS) + return FS == MaybeFS ? BaseFS : MaybeFS; + if (!FileMgr) + return FS ? BaseFS : createVFSFromCompilerInvocation(CI, Diags); + + llvm::vfs::FileSystem &FMFS = FileMgr->getVirtualFileSystem(); + return FS == &FMFS ? BaseFS : &FMFS; +} + +void ASTUnit::computeAndInitializeFS( + const CompilerInvocation &CI, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr MaybeFS) { + initializeBaseFS(computeBaseFS(CI, Diags, std::move(MaybeFS))); +} + +void ASTUnit::remapFiles(Optional> RemappedFiles) { + if (!RemappedFiles || RemappedFiles->empty()) + return; + + // Add files to the in-memory filesystem in reverse order to ensure the last + // mapping "wins". This works because InMemoryFileSystem::addFile ignores + // subsequent calls. + for (auto &File : llvm::reverse(*RemappedFiles)) + RemappedFilesFS->addFile(File.first, 0, std::move(File.second)); +} Index: clang/lib/Frontend/PrecompiledPreamble.cpp =================================================================== --- clang/lib/Frontend/PrecompiledPreamble.cpp +++ clang/lib/Frontend/PrecompiledPreamble.cpp @@ -388,6 +388,7 @@ Diagnostics.Reset(); ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts()); + // FIXME: should this be deleted? Can we trust that the caller has done this? VFS = createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS); @@ -404,6 +405,9 @@ Clang->getLangOpts().CompilingPCH = true; // Remap the main source file to the preamble buffer. + // + // Note: the buffer needs to be overridden in the SourceManager so that the + // preamble is serialized the AST file. StringRef MainFilePath = FrontendOpts.Inputs[0].getFile(); auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy( MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath); @@ -500,11 +504,11 @@ llvm_unreachable("Unhandled storage kind"); } -bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation, - const llvm::MemoryBufferRef &MainFileBuffer, - PreambleBounds Bounds, - llvm::vfs::FileSystem &VFS) const { - +bool PrecompiledPreamble::CanReuse( + const CompilerInvocation &Invocation, + const llvm::MemoryBufferRef &MainFileBuffer, PreambleBounds Bounds, + llvm::vfs::FileSystem &VFS, + llvm::vfs::FileSystem *OverriddenFilesFS) const { assert( Bounds.Size <= MainFileBuffer.getBufferSize() && "Buffer is too large. Bounds were calculated from a different buffer?"); @@ -564,6 +568,15 @@ // Check whether anything has changed. for (const auto &F : FilesInPreamble) { + if (OverriddenFilesFS) { + std::unique_ptr Buffer; + if (moveOnNoError(OverriddenFilesFS->getBufferForFile(F.first()), + Buffer)) { + if (PreambleFileHash::createForMemoryBuffer(*Buffer) != F.second) + return false; + continue; + } + } auto OverridenFileBuffer = OverridenFileBuffers.find(F.first()); if (OverridenFileBuffer != OverridenFileBuffers.end()) { // The file's buffer was remapped and the file was not found in VFS. @@ -804,7 +817,7 @@ auto &PreprocessorOpts = CI.getPreprocessorOpts(); - // Remap main file to point to MainFileBuffer. + // Remap main file to point to MainFileBuffer (instead of the preamble). auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile(); PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer); Index: clang/tools/libclang/CIndex.cpp =================================================================== --- clang/tools/libclang/CIndex.cpp +++ clang/tools/libclang/CIndex.cpp @@ -3564,11 +3564,10 @@ llvm::CrashRecoveryContextCleanupRegistrar> RemappedCleanup(RemappedFiles.get()); - for (auto &UF : unsaved_files) { - std::unique_ptr MB = - llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename); - RemappedFiles->push_back(std::make_pair(UF.Filename, MB.release())); - } + for (auto &UF : unsaved_files) + RemappedFiles->push_back(std::make_pair( + UF.Filename, + llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename))); std::unique_ptr> Args( new std::vector()); @@ -3629,8 +3628,9 @@ Args->data(), Args->data() + Args->size(), CXXIdx->getPCHContainerOperations(), Diags, CXXIdx->getClangResourcesPath(), CXXIdx->getOnlyLocalDecls(), - CaptureDiagnostics, *RemappedFiles.get(), PrecompilePreambleAfterNParses, - TUKind, CacheCodeCompletionResults, IncludeBriefCommentsInCodeCompletion, + CaptureDiagnostics, std::move(*RemappedFiles), + PrecompilePreambleAfterNParses, TUKind, CacheCodeCompletionResults, + IncludeBriefCommentsInCodeCompletion, /*AllowPCHWithCompilerErrors=*/true, SkipFunctionBodies, SingleFileParse, /*UserFilesAreVolatile=*/true, ForSerialization, RetainExcludedCB, CXXIdx->getPCHContainerOperations()->getRawReader().getFormat(), @@ -4214,14 +4214,13 @@ llvm::CrashRecoveryContextCleanupRegistrar> RemappedCleanup(RemappedFiles.get()); - for (auto &UF : unsaved_files) { - std::unique_ptr MB = - llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename); - RemappedFiles->push_back(std::make_pair(UF.Filename, MB.release())); - } + for (auto &UF : unsaved_files) + RemappedFiles->push_back(std::make_pair( + UF.Filename, + llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename))); if (!CXXUnit->Reparse(CXXIdx->getPCHContainerOperations(), - *RemappedFiles.get())) + std::move(*RemappedFiles))) return CXError_Success; if (isASTReadError(CXXUnit)) return CXError_ASTReadError; Index: clang/tools/libclang/CIndexCodeCompletion.cpp =================================================================== --- clang/tools/libclang/CIndexCodeCompletion.cpp +++ clang/tools/libclang/CIndexCodeCompletion.cpp @@ -247,7 +247,7 @@ /// AllocatedCXCodeCompleteResults outlives the CXTranslationUnit, so we can /// not rely on the StringPool in the TU. struct AllocatedCXCodeCompleteResults : public CXCodeCompleteResults { - AllocatedCXCodeCompleteResults(IntrusiveRefCntPtr FileMgr); + AllocatedCXCodeCompleteResults(); ~AllocatedCXCodeCompleteResults(); /// Diagnostics produced while performing code completion. @@ -354,8 +354,7 @@ /// Used for debugging purposes only. static std::atomic CodeCompletionResultObjects; -AllocatedCXCodeCompleteResults::AllocatedCXCodeCompleteResults( - IntrusiveRefCntPtr FileMgr) +AllocatedCXCodeCompleteResults::AllocatedCXCodeCompleteResults() : CXCodeCompleteResults(), DiagOpts(new DiagnosticOptions), Diag(new DiagnosticsEngine( IntrusiveRefCntPtr(new DiagnosticIDs), &*DiagOpts)), @@ -717,21 +716,19 @@ ASTUnit::ConcurrencyCheck Check(*AST); // Perform the remapping of source files. - SmallVector RemappedFiles; + std::vector RemappedFiles; - for (auto &UF : unsaved_files) { - std::unique_ptr MB = - llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename); - RemappedFiles.push_back(std::make_pair(UF.Filename, MB.release())); - } + for (auto &UF : unsaved_files) + RemappedFiles.push_back(std::make_pair( + UF.Filename, + llvm::MemoryBuffer::getMemBufferCopy(getContents(UF), UF.Filename))); if (EnableLogging) { // FIXME: Add logging. } // Parse the resulting source file to find code-completion results. - AllocatedCXCodeCompleteResults *Results = new AllocatedCXCodeCompleteResults( - &AST->getFileManager()); + AllocatedCXCodeCompleteResults *Results = new AllocatedCXCodeCompleteResults; Results->Results = nullptr; Results->NumResults = 0; @@ -754,8 +751,8 @@ *CXXIdx, LibclangInvocationReporter::OperationKind::CompletionOperation, TU->ParsingOptions, CArgs, CompletionInvocation, unsaved_files); AST->CodeComplete( - complete_filename, complete_line, complete_column, RemappedFiles, - (options & CXCodeComplete_IncludeMacros), + complete_filename, complete_line, complete_column, + std::move(RemappedFiles), (options & CXCodeComplete_IncludeMacros), (options & CXCodeComplete_IncludeCodePatterns), IncludeBriefComments, Capture, CXXIdx->getPCHContainerOperations(), *Results->Diag, Results->LangOpts, Results->Diagnostics, Results->TemporaryBuffers); Index: clang/unittests/Frontend/PCHPreambleTest.cpp =================================================================== --- clang/unittests/Frontend/PCHPreambleTest.cpp +++ clang/unittests/Frontend/PCHPreambleTest.cpp @@ -85,11 +85,6 @@ CI->getTargetOpts().Triple = "i386-unknown-linux-gnu"; - CI->getPreprocessorOpts().RemappedFileBuffers = GetRemappedFiles(); - - PreprocessorOptions &PPOpts = CI->getPreprocessorOpts(); - PPOpts.RemappedFilesKeepOriginalName = true; - IntrusiveRefCntPtr Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions, new DiagnosticConsumer)); @@ -97,7 +92,8 @@ std::unique_ptr AST = ASTUnit::LoadFromCompilerInvocation( CI, PCHContainerOpts, Diags, FileMgr, - /*PrecompilePreambleAfterNParses=*/1); + /*PrecompilePreambleAfterNParses=*/1, /*UserFilesAreVolatile=*/false, + GetRemappedFiles()); return AST; } @@ -111,9 +107,8 @@ } private: - std::vector> - GetRemappedFiles() const { - std::vector> Remapped; + std::vector GetRemappedFiles() const { + std::vector Remapped; for (const auto &RemappedFile : RemappedFiles) { std::unique_ptr buf = MemoryBuffer::getMemBufferCopy( RemappedFile.second, RemappedFile.first());