Index: clang-tools-extra/trunk/clangd/ClangdLSPServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdLSPServer.h +++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h @@ -31,7 +31,7 @@ /// loaded only from \p CompileCommandsDir. Otherwise, clangd will look /// for compile_commands.json in all parent directories of each file. ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount, - bool SnippetCompletions, + bool StorePreamblesInMemory, bool SnippetCompletions, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir); Index: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp @@ -235,11 +235,13 @@ } ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount, + bool StorePreamblesInMemory, bool SnippetCompletions, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir) : Out(Out), CDB(/*Logger=*/Out, std::move(CompileCommandsDir)), Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, + StorePreamblesInMemory, clangd::CodeCompleteOptions( /*EnableSnippetsAndCodePatterns=*/SnippetCompletions), /*Logger=*/Out, ResourceDir) {} Index: clang-tools-extra/trunk/clangd/ClangdServer.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.h +++ clang-tools-extra/trunk/clangd/ClangdServer.h @@ -205,10 +205,14 @@ /// worker thread. Therefore, instances of \p DiagConsumer must properly /// synchronize access to shared state. /// + /// \p StorePreamblesInMemory defines whether the Preambles generated by + /// clangd are stored in-memory or on disk. + /// /// Various messages are logged using \p Logger. ClangdServer(GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, + bool StorePreamblesInMemory, clangd::CodeCompleteOptions CodeCompleteOpts, clangd::Logger &Logger, llvm::Optional ResourceDir = llvm::None); @@ -325,6 +329,7 @@ // If set, this represents the workspace path. llvm::Optional RootPath; std::shared_ptr PCHs; + bool StorePreamblesInMemory; clangd::CodeCompleteOptions CodeCompleteOpts; /// Used to serialize diagnostic callbacks. /// FIXME(ibiryukov): get rid of an extra map and put all version counters Index: clang-tools-extra/trunk/clangd/ClangdServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdServer.cpp @@ -169,17 +169,16 @@ Worker.join(); } -ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB, - DiagnosticsConsumer &DiagConsumer, - FileSystemProvider &FSProvider, - unsigned AsyncThreadsCount, - clangd::CodeCompleteOptions CodeCompleteOpts, - clangd::Logger &Logger, - llvm::Optional ResourceDir) +ClangdServer::ClangdServer( + GlobalCompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, + FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, + bool StorePreamblesInMemory, clangd::CodeCompleteOptions CodeCompleteOpts, + clangd::Logger &Logger, llvm::Optional ResourceDir) : Logger(Logger), CDB(CDB), DiagConsumer(DiagConsumer), FSProvider(FSProvider), ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()), PCHs(std::make_shared()), + StorePreamblesInMemory(StorePreamblesInMemory), CodeCompleteOpts(CodeCompleteOpts), WorkScheduler(AsyncThreadsCount) {} void ClangdServer::setRootPath(PathRef RootPath) { @@ -193,8 +192,8 @@ DocVersion Version = DraftMgr.updateDraft(File, Contents); auto TaggedFS = FSProvider.getTaggedFileSystem(File); - std::shared_ptr Resources = - Units.getOrCreateFile(File, ResourceDir, CDB, PCHs, Logger); + std::shared_ptr Resources = Units.getOrCreateFile( + File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); return scheduleReparseAndDiags(File, VersionedDraft{Version, Contents.str()}, std::move(Resources), std::move(TaggedFS)); } @@ -211,8 +210,8 @@ "forceReparse() was called for non-added document"); auto TaggedFS = FSProvider.getTaggedFileSystem(File); - auto Recreated = Units.recreateFileIfCompileCommandChanged(File, ResourceDir, - CDB, PCHs, Logger); + auto Recreated = Units.recreateFileIfCompileCommandChanged( + File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); // Note that std::future from this cleanup action is ignored. scheduleCancelRebuild(std::move(Recreated.RemovedFile)); Index: clang-tools-extra/trunk/clangd/ClangdUnit.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnit.h +++ clang-tools-extra/trunk/clangd/ClangdUnit.h @@ -143,10 +143,12 @@ // deferRebuild will hold references to it. static std::shared_ptr Create(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger); private: CppFile(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger); public: @@ -222,6 +224,7 @@ Path FileName; tooling::CompileCommand Command; + bool StorePreamblesInMemory; /// Mutex protects all fields, declared below it, FileName and Command are not /// mutated. Index: clang-tools-extra/trunk/clangd/ClangdUnit.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnit.cpp +++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp @@ -235,7 +235,7 @@ // NOTE: we use Buffer.get() when adding remapped files, so we have to make // sure it will be released if no error is emitted. if (Preamble) { - Preamble->AddImplicitPreamble(*CI, Buffer.get()); + Preamble->AddImplicitPreamble(*CI, VFS, Buffer.get()); } else { CI->getPreprocessorOpts().addRemappedFile( CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get()); @@ -1137,16 +1137,20 @@ std::shared_ptr CppFile::Create(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger) { - return std::shared_ptr( - new CppFile(FileName, std::move(Command), std::move(PCHs), Logger)); + return std::shared_ptr(new CppFile(FileName, std::move(Command), + StorePreamblesInMemory, + std::move(PCHs), Logger)); } CppFile::CppFile(PathRef FileName, tooling::CompileCommand Command, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger) - : FileName(FileName), Command(std::move(Command)), RebuildCounter(0), + : FileName(FileName), Command(std::move(Command)), + StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0), RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger) { std::lock_guard Lock(Mutex); @@ -1290,6 +1294,7 @@ CppFilePreambleCallbacks SerializedDeclsCollector; auto BuiltPreamble = PrecompiledPreamble::Build( *CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs, + /*StoreInMemory=*/That->StorePreamblesInMemory, SerializedDeclsCollector); if (BuiltPreamble) { Index: clang-tools-extra/trunk/clangd/ClangdUnitStore.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnitStore.h +++ clang-tools-extra/trunk/clangd/ClangdUnitStore.h @@ -25,9 +25,11 @@ /// Thread-safe mapping from FileNames to CppFile. class CppFileCollection { public: - std::shared_ptr getOrCreateFile( - PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, - std::shared_ptr PCHs, clangd::Logger &Logger) { + std::shared_ptr + getOrCreateFile(PathRef File, PathRef ResourceDir, + GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory, + std::shared_ptr PCHs, + clangd::Logger &Logger) { std::lock_guard Lock(Mutex); auto It = OpenedFiles.find(File); @@ -36,6 +38,7 @@ It = OpenedFiles .try_emplace(File, CppFile::Create(File, std::move(Command), + StorePreamblesInMemory, std::move(PCHs), Logger)) .first; } @@ -58,7 +61,8 @@ /// will be returned in RecreateResult.RemovedFile. RecreateResult recreateFileIfCompileCommandChanged( PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, - std::shared_ptr PCHs, clangd::Logger &Logger); + bool StorePreamblesInMemory, std::shared_ptr PCHs, + clangd::Logger &Logger); std::shared_ptr getFile(PathRef File) { std::lock_guard Lock(Mutex); Index: clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp +++ clang-tools-extra/trunk/clangd/ClangdUnitStore.cpp @@ -29,7 +29,8 @@ CppFileCollection::RecreateResult CppFileCollection::recreateFileIfCompileCommandChanged( PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, - std::shared_ptr PCHs, clangd::Logger &Logger) { + bool StorePreamblesInMemory, std::shared_ptr PCHs, + clangd::Logger &Logger) { auto NewCommand = getCompileCommand(CDB, File, ResourceDir); std::lock_guard Lock(Mutex); @@ -40,13 +41,15 @@ if (It == OpenedFiles.end()) { It = OpenedFiles .try_emplace(File, CppFile::Create(File, std::move(NewCommand), + StorePreamblesInMemory, std::move(PCHs), Logger)) .first; } else if (!compileCommandsAreEqual(It->second->getCompileCommand(), NewCommand)) { Result.RemovedFile = std::move(It->second); It->second = - CppFile::Create(File, std::move(NewCommand), std::move(PCHs), Logger); + CppFile::Create(File, std::move(NewCommand), StorePreamblesInMemory, + std::move(PCHs), Logger); } Result.FileInCollection = It->second; return Result; Index: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp =================================================================== --- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp +++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp @@ -24,6 +24,10 @@ using namespace clang; using namespace clang::clangd; +namespace { +enum class PCHStorageFlag { Disk, Memory }; +} + static llvm::cl::opt CompileCommandsDir( "compile-commands-dir", llvm::cl::desc("Specify a path to look for compile_commands.json. If path " @@ -45,6 +49,15 @@ PrettyPrint("pretty", llvm::cl::desc("Pretty-print JSON output"), llvm::cl::init(false)); +static llvm::cl::opt PCHStorage( + "pch-storage", + llvm::cl::desc("Storing PCHs in memory increases memory usages, but may " + "improve performance"), + llvm::cl::values( + clEnumValN(PCHStorageFlag::Disk, "disk", "store PCHs on disk"), + clEnumValN(PCHStorageFlag::Memory, "memory", "store PCHs in memory")), + llvm::cl::init(PCHStorageFlag::Disk)); + static llvm::cl::opt RunSynchronously( "run-synchronously", llvm::cl::desc("Parse on main thread. If set, -j is ignored"), @@ -127,6 +140,16 @@ CompileCommandsDirPath = CompileCommandsDir; } + bool StorePreamblesInMemory; + switch (PCHStorage) { + case PCHStorageFlag::Memory: + StorePreamblesInMemory = true; + break; + case PCHStorageFlag::Disk: + StorePreamblesInMemory = false; + break; + } + llvm::Optional ResourceDirRef = None; if (!ResourceDir.empty()) ResourceDirRef = ResourceDir; @@ -135,8 +158,9 @@ llvm::sys::ChangeStdinToBinary(); // Initialize and run ClangdLSPServer. - ClangdLSPServer LSPServer(Out, WorkerThreadsCount, EnableSnippets, - ResourceDirRef, CompileCommandsDirPath); + ClangdLSPServer LSPServer(Out, WorkerThreadsCount, StorePreamblesInMemory, + EnableSnippets, ResourceDirRef, + CompileCommandsDirPath); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); return LSPServer.run(std::cin) ? 0 : NoShutdownRequestErrorCode; Index: clang-tools-extra/trunk/test/clangd/completion.test =================================================================== --- clang-tools-extra/trunk/test/clangd/completion.test +++ clang-tools-extra/trunk/test/clangd/completion.test @@ -1,4 +1,5 @@ # RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s +# RUN: clangd -pretty -run-synchronously -pch-storage=memory < %s | FileCheck -strict-whitespace %s # It is absolutely vital that this file has CRLF line endings. # Content-Length: 125 Index: clang-tools-extra/trunk/test/clangd/diagnostics-preamble.test =================================================================== --- clang-tools-extra/trunk/test/clangd/diagnostics-preamble.test +++ clang-tools-extra/trunk/test/clangd/diagnostics-preamble.test @@ -1,4 +1,5 @@ # RUN: clangd -pretty -run-synchronously < %s | FileCheck -strict-whitespace %s +# RUN: clangd -pretty -run-synchronously -pch-storage=memory < %s | FileCheck -strict-whitespace %s # It is absolutely vital that this file has CRLF line endings. # Content-Length: 125 Index: clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp =================================================================== --- clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp +++ clang-tools-extra/trunk/unittests/clangd/ClangdTests.cpp @@ -332,6 +332,7 @@ ErrorCheckingDiagConsumer DiagConsumer; MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); for (const auto &FileWithContents : ExtraFiles) @@ -396,6 +397,7 @@ ErrorCheckingDiagConsumer DiagConsumer; MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -442,6 +444,7 @@ MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -490,7 +493,9 @@ MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); // Run ClangdServer synchronously. ClangdServer Server(CDB, DiagConsumer, FS, - /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(), + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); auto FooCpp = getVirtualTestFilePath("foo.cpp"); @@ -524,7 +529,9 @@ "-stdlib=libstdc++"}); // Run ClangdServer synchronously. ClangdServer Server(CDB, DiagConsumer, FS, - /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(), + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); // Just a random gcc version string @@ -573,7 +580,9 @@ ErrorCheckingDiagConsumer DiagConsumer; MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, - /*AsyncThreadsCount=*/0, clangd::CodeCompleteOptions(), + /*AsyncThreadsCount=*/0, + /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); // No need to sync reparses, because reparses are performed on the calling // thread to true. @@ -641,6 +650,7 @@ MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -701,7 +711,8 @@ clangd::CodeCompleteOptions Opts; Opts.Limit = 2; ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), - Opts, EmptyLogger::getInstance()); + /*StorePreamblesInMemory=*/true, Opts, + EmptyLogger::getInstance()); auto FooCpp = getVirtualTestFilePath("foo.cpp"); FS.Files[FooCpp] = ""; @@ -796,7 +807,8 @@ auto TestWithOpts = [&](clangd::CodeCompleteOptions Opts) { ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), - Opts, EmptyLogger::getInstance()); + /*StorePreamblesInMemory=*/true, Opts, + EmptyLogger::getInstance()); // No need to sync reparses here as there are no asserts on diagnostics (or // other async operations). Server.addDocument(FooCpp, GlobalCompletion.Text); @@ -989,6 +1001,7 @@ { MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -1149,6 +1162,7 @@ MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); ClangdServer Server(CDB, DiagConsumer, FS, getDefaultAsyncThreadsCount(), + /*StorePreamblesInMemory=*/true, clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); @@ -1275,7 +1289,8 @@ std::move(StartSecondReparsePromise)); MockCompilationDatabase CDB(/*AddFreestandingFlag=*/true); - ClangdServer Server(CDB, DiagConsumer, FS, 4, clangd::CodeCompleteOptions(), + ClangdServer Server(CDB, DiagConsumer, FS, 4, /*StorePreamblesInMemory=*/true, + clangd::CodeCompleteOptions(), EmptyLogger::getInstance()); Server.addDocument(FooCpp, SourceContentsWithErrors); StartSecondReparse.wait();