Index: clangd/ClangdLSPServer.h =================================================================== --- clangd/ClangdLSPServer.h +++ clangd/ClangdLSPServer.h @@ -94,7 +94,7 @@ // Various ClangdServer parameters go here. It's important they're created // before ClangdServer. - DirectoryBasedGlobalCompilationDatabase CDB; + ExtraFlagsCompilationDatabase CDB; RealFileSystemProvider FSProvider; // Server must be the last member of the class to allow its destructor to exit Index: clangd/ClangdLSPServer.cpp =================================================================== --- clangd/ClangdLSPServer.cpp +++ clangd/ClangdLSPServer.cpp @@ -234,12 +234,32 @@ C.reply(Result ? URI::fromFile(*Result).uri : ""); } +// Creates the base compilation database, which must always return a command. +// The per-file extra flags are overlaid on top of this. +static std::unique_ptr +CreateCDB(clangd::Logger &Out, llvm::Optional CompileCommandsDir) { + using namespace tooling; + std::unique_ptr Primary; + if (CompileCommandsDir) { + std::string Err; + Primary = CompilationDatabase::loadFromDirectory(*CompileCommandsDir, Err); + if (!Primary) + Out.log(llvm::Twine("Failed to load compilation database from ") + + *CompileCommandsDir + ", falling back to default: " + Err); + } + if (!Primary) + Primary = llvm::make_unique(); + return llvm::make_unique( + std::move(Primary), llvm::make_unique( + ".", ArrayRef{})); +} + ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional ResourceDir, llvm::Optional CompileCommandsDir) - : Out(Out), CDB(/*Logger=*/Out, std::move(CompileCommandsDir)), + : Out(Out), CDB(CreateCDB(Out, std::move(CompileCommandsDir))), Server(CDB, /*DiagConsumer=*/*this, FSProvider, AsyncThreadsCount, StorePreamblesInMemory, CCOpts, /*Logger=*/Out, ResourceDir) {} Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -12,7 +12,6 @@ #include "ClangdUnitStore.h" #include "DraftStore.h" -#include "GlobalCompilationDatabase.h" #include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Core/Replacement.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" @@ -206,7 +205,7 @@ /// clangd are stored in-memory or on disk. /// /// Various messages are logged using \p Logger. - ClangdServer(GlobalCompilationDatabase &CDB, + ClangdServer(const tooling::CompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, bool StorePreamblesInMemory, @@ -230,7 +229,7 @@ /// associated resources are freed. std::future removeDocument(PathRef File); /// Force \p File to be reparsed using the latest contents. - /// Will also check if CompileCommand, provided by GlobalCompilationDatabase + /// Will also check if CompileCommand, provided by CompilationDatabase /// for \p File has changed. If it has, will remove currently stored Preamble /// and AST and rebuild them from scratch. std::future forceReparse(PathRef File); @@ -317,7 +316,7 @@ std::future scheduleCancelRebuild(std::shared_ptr Resources); clangd::Logger &Logger; - GlobalCompilationDatabase &CDB; + std::unique_ptr CDB; DiagnosticsConsumer &DiagConsumer; FileSystemProvider &FSProvider; DraftStore DraftMgr; Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -8,9 +8,11 @@ //===-------------------------------------------------------------------===// #include "ClangdServer.h" +#include "GlobalCompilationDatabase.h" #include "clang/Format/Format.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" +#include "clang/Tooling/CommonOptionsParser.h" #include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Refactoring/RefactoringResultConsumer.h" #include "clang/Tooling/Refactoring/Rename/RenamingAction.h" @@ -169,7 +171,36 @@ Worker.join(); } -ClangdServer::ClangdServer(GlobalCompilationDatabase &CDB, +static std::unique_ptr +CreateCDB(const tooling::CompilationDatabase &CDB, + llvm::Optional ResourceDir) { + // Create an owned version of CDB to use ArgumentsAdjustingCompilations. + struct OwnedCDB : public tooling::CompilationDatabase { + OwnedCDB(const tooling::CompilationDatabase &Ref) : Ref(Ref) {} + const tooling::CompilationDatabase &Ref; + std::vector + getCompileCommands(StringRef P) const override { + return Ref.getCompileCommands(P); + } + }; + std::unique_ptr Base(new OwnedCDB(CDB)); + // FIXME: Instead of falling back to fixed arguments, we should refuse to + // parse files without commands. The caller can set a fallback policy. + Base = llvm::make_unique( + std::move(Base), llvm::make_unique( + ".", ArrayRef{})); + auto Result = llvm::make_unique( + std::move(Base)); + Result->appendArgumentsAdjuster(tooling::getClangSyntaxOnlyAdjuster()); + Result->appendArgumentsAdjuster(tooling::getClangStripOutputAdjuster()); + Result->appendArgumentsAdjuster(tooling::getInsertArgumentAdjuster( + {"-resource-dir=" + + (ResourceDir ? ResourceDir->str() : getStandardResourceDir())}, + tooling::ArgumentInsertPosition::BEGIN)); + return std::move(Result); +} + +ClangdServer::ClangdServer(const tooling::CompilationDatabase &CDB, DiagnosticsConsumer &DiagConsumer, FileSystemProvider &FSProvider, unsigned AsyncThreadsCount, @@ -177,9 +208,8 @@ const clangd::CodeCompleteOptions &CodeCompleteOpts, clangd::Logger &Logger, llvm::Optional ResourceDir) - : Logger(Logger), CDB(CDB), DiagConsumer(DiagConsumer), - FSProvider(FSProvider), - ResourceDir(ResourceDir ? ResourceDir->str() : getStandardResourceDir()), + : Logger(Logger), CDB(CreateCDB(CDB, ResourceDir)), + DiagConsumer(DiagConsumer), FSProvider(FSProvider), PCHs(std::make_shared()), StorePreamblesInMemory(StorePreamblesInMemory), CodeCompleteOpts(CodeCompleteOpts), WorkScheduler(AsyncThreadsCount) {} @@ -195,8 +225,8 @@ DocVersion Version = DraftMgr.updateDraft(File, Contents); auto TaggedFS = FSProvider.getTaggedFileSystem(File); - std::shared_ptr Resources = Units.getOrCreateFile( - File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); + std::shared_ptr Resources = + Units.getOrCreateFile(File, *CDB, StorePreamblesInMemory, PCHs, Logger); return scheduleReparseAndDiags(File, VersionedDraft{Version, Contents.str()}, std::move(Resources), std::move(TaggedFS)); } @@ -214,7 +244,7 @@ auto TaggedFS = FSProvider.getTaggedFileSystem(File); auto Recreated = Units.recreateFileIfCompileCommandChanged( - File, ResourceDir, CDB, StorePreamblesInMemory, PCHs, Logger); + File, *CDB, StorePreamblesInMemory, PCHs, Logger); // Note that std::future from this cleanup action is ignored. scheduleCancelRebuild(std::move(Recreated.RemovedFile)); Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -1153,6 +1153,8 @@ : FileName(FileName), Command(std::move(Command)), StorePreamblesInMemory(StorePreamblesInMemory), RebuildCounter(0), RebuildInProgress(false), PCHs(std::move(PCHs)), Logger(Logger) { + Logger.log(llvm::Twine("File ") + FileName + + " has command: " + llvm::join(this->Command.CommandLine, " ")); std::lock_guard Lock(Mutex); LatestAvailablePreamble = nullptr; Index: clangd/ClangdUnitStore.h =================================================================== --- clangd/ClangdUnitStore.h +++ clangd/ClangdUnitStore.h @@ -13,7 +13,6 @@ #include #include "ClangdUnit.h" -#include "GlobalCompilationDatabase.h" #include "Path.h" #include "clang/Tooling/CompilationDatabase.h" @@ -26,20 +25,21 @@ class CppFileCollection { public: std::shared_ptr - getOrCreateFile(PathRef File, PathRef ResourceDir, - GlobalCompilationDatabase &CDB, bool StorePreamblesInMemory, + getOrCreateFile(PathRef File, tooling::CompilationDatabase &CDB, + bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger) { std::lock_guard Lock(Mutex); auto It = OpenedFiles.find(File); if (It == OpenedFiles.end()) { - auto Command = getCompileCommand(CDB, File, ResourceDir); - + auto Commands = CDB.getCompileCommands(File); + assert(!Commands.empty() && "Expected at least a fallback command!"); It = OpenedFiles - .try_emplace(File, CppFile::Create(File, std::move(Command), - StorePreamblesInMemory, - std::move(PCHs), Logger)) + .try_emplace(File, + CppFile::Create(File, std::move(Commands.front()), + StorePreamblesInMemory, + std::move(PCHs), Logger)) .first; } return It->second; @@ -60,7 +60,7 @@ /// If a currently stored CppFile had to be replaced, the previous instance /// will be returned in RecreateResult.RemovedFile. RecreateResult recreateFileIfCompileCommandChanged( - PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, + PathRef File, tooling::CompilationDatabase &CDB, bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger); @@ -78,9 +78,6 @@ std::shared_ptr removeIfPresent(PathRef File); private: - tooling::CompileCommand getCompileCommand(GlobalCompilationDatabase &CDB, - PathRef File, PathRef ResourceDir); - bool compileCommandsAreEqual(tooling::CompileCommand const &LHS, tooling::CompileCommand const &RHS); Index: clangd/ClangdUnitStore.cpp =================================================================== --- clangd/ClangdUnitStore.cpp +++ clangd/ClangdUnitStore.cpp @@ -28,10 +28,12 @@ CppFileCollection::RecreateResult CppFileCollection::recreateFileIfCompileCommandChanged( - PathRef File, PathRef ResourceDir, GlobalCompilationDatabase &CDB, + PathRef File, tooling::CompilationDatabase &CDB, bool StorePreamblesInMemory, std::shared_ptr PCHs, clangd::Logger &Logger) { - auto NewCommand = getCompileCommand(CDB, File, ResourceDir); + auto Commands = CDB.getCompileCommands(File); + assert(!Commands.empty() && "Expected at least a fallback command!"); + auto &NewCommand = Commands.front(); std::lock_guard Lock(Mutex); @@ -55,21 +57,6 @@ return Result; } -tooling::CompileCommand -CppFileCollection::getCompileCommand(GlobalCompilationDatabase &CDB, - PathRef File, PathRef ResourceDir) { - std::vector Commands = CDB.getCompileCommands(File); - if (Commands.empty()) - // Add a fake command line if we know nothing. - Commands.push_back(getDefaultCompileCommand(File)); - - // Inject the resource dir. - // FIXME: Don't overwrite it if it's already there. - Commands.front().CommandLine.push_back("-resource-dir=" + - std::string(ResourceDir)); - return std::move(Commands.front()); -} - bool CppFileCollection::compileCommandsAreEqual( tooling::CompileCommand const &LHS, tooling::CompileCommand const &RHS) { // tooling::CompileCommand.Output is ignored, it's not relevant for clangd. Index: clangd/GlobalCompilationDatabase.h =================================================================== --- clangd/GlobalCompilationDatabase.h +++ clangd/GlobalCompilationDatabase.h @@ -6,73 +6,77 @@ // License. See LICENSE.TXT for details. // //===---------------------------------------------------------------------===// +// +// Defines CompilationDatabase that help clangd determine compile commands. +// We plug together a few of these, depending on configuration. +// +//===---------------------------------------------------------------------===// #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H #include "Path.h" +#include "clang/Tooling/CompilationDatabase.h" #include "llvm/ADT/StringMap.h" #include #include #include namespace clang { - -namespace tooling { -class CompilationDatabase; -struct CompileCommand; -} // namespace tooling - namespace clangd { - class Logger; -/// Returns a default compile command to use for \p File. -tooling::CompileCommand getDefaultCompileCommand(PathRef File); - -/// Provides compilation arguments used for parsing C and C++ files. -class GlobalCompilationDatabase { +/// Provides compile commands for any file by scanning parent directories +/// looking for compilation databases. These are cached for efficiency. +class GlobalCompilationDatabase : public tooling::CompilationDatabase { public: - virtual ~GlobalCompilationDatabase() = default; + std::vector + getCompileCommands(PathRef File) const override; - virtual std::vector - getCompileCommands(PathRef File) = 0; +private: + tooling::CompilationDatabase *getCompilationDatabase(PathRef File) const; - /// FIXME(ibiryukov): add facilities to track changes to compilation flags of - /// existing targets. + mutable std::mutex Mutex; + /// Caches compilation databases from directories. Dir -> DB or null. + mutable llvm::StringMap> + Cache; }; -/// Gets compile args from tooling::CompilationDatabases built for parent -/// directories. -class DirectoryBasedGlobalCompilationDatabase - : public GlobalCompilationDatabase { +/// Adds extra flags into the compile commands of another database. +/// The extra flags are configured per file. +class ExtraFlagsCompilationDatabase : public tooling::CompilationDatabase { public: - DirectoryBasedGlobalCompilationDatabase( - clangd::Logger &Logger, llvm::Optional CompileCommandsDir); + ExtraFlagsCompilationDatabase( + std::unique_ptr Inner) + : Inner(std::move(Inner)) {} std::vector - getCompileCommands(PathRef File) override; + getCompileCommands(PathRef File) const override; - void setExtraFlagsForFile(PathRef File, std::vector ExtraFlags); + void setExtraFlagsForFile(PathRef File, std::vector Flags); private: - tooling::CompilationDatabase *getCompilationDatabase(PathRef File); - tooling::CompilationDatabase *tryLoadDatabaseFromPath(PathRef File); + mutable std::mutex Mutex; + llvm::StringMap> ExtraFlags; + std::unique_ptr Inner; +}; - std::mutex Mutex; - /// Caches compilation databases loaded from directories(keys are - /// directories). - llvm::StringMap> - CompilationDatabases; +/// Attempts to use a primary compilation database, falling back to a second +/// one if the first returns no commands. +class FallbackCompilationDatabase : public tooling::CompilationDatabase { +public: + FallbackCompilationDatabase( + std::unique_ptr Primary, + std::unique_ptr Fallback) + : Primary(std::move(Primary)), Fallback(std::move(Fallback)) {} - /// Stores extra flags per file. - llvm::StringMap> ExtraFlagsForFile; - /// Used for logging. - clangd::Logger &Logger; - /// Used for command argument pointing to folder where compile_commands.json - /// is located. - llvm::Optional CompileCommandsDir; + std::vector + getCompileCommands(PathRef File) const override; + +private: + std::unique_ptr Primary, Fallback; }; + } // namespace clangd } // namespace clang Index: clangd/GlobalCompilationDatabase.cpp =================================================================== --- clangd/GlobalCompilationDatabase.cpp +++ clangd/GlobalCompilationDatabase.cpp @@ -15,107 +15,66 @@ namespace clang { namespace clangd { +using namespace tooling; +namespace path = llvm::sys::path; -static void addExtraFlags(tooling::CompileCommand &Command, - const std::vector &ExtraFlags) { - if (ExtraFlags.empty()) - return; - assert(Command.CommandLine.size() >= 2 && - "Expected a command line containing at least 2 arguments, the " - "compiler binary and the output file"); - // The last argument of CommandLine is the name of the input file. - // Add ExtraFlags before it. - auto It = Command.CommandLine.end(); - --It; - Command.CommandLine.insert(It, ExtraFlags.begin(), ExtraFlags.end()); -} - -tooling::CompileCommand getDefaultCompileCommand(PathRef File) { - std::vector CommandLine{"clang", "-fsyntax-only", File.str()}; - return tooling::CompileCommand(llvm::sys::path::parent_path(File), - llvm::sys::path::filename(File), CommandLine, - /*Output=*/""); -} - -DirectoryBasedGlobalCompilationDatabase:: - DirectoryBasedGlobalCompilationDatabase( - clangd::Logger &Logger, llvm::Optional CompileCommandsDir) - : Logger(Logger), CompileCommandsDir(std::move(CompileCommandsDir)) {} - -std::vector -DirectoryBasedGlobalCompilationDatabase::getCompileCommands(PathRef File) { - std::vector Commands; - - auto CDB = getCompilationDatabase(File); - if (CDB) +std::vector +GlobalCompilationDatabase::getCompileCommands(PathRef File) const { + std::vector Commands; + if (auto CDB = getCompilationDatabase(File)) Commands = CDB->getCompileCommands(File); - if (Commands.empty()) - Commands.push_back(getDefaultCompileCommand(File)); - - auto It = ExtraFlagsForFile.find(File); - if (It != ExtraFlagsForFile.end()) { - // Append the user-specified flags to the compile commands. - for (tooling::CompileCommand &Command : Commands) - addExtraFlags(Command, It->second); - } - return Commands; } -void DirectoryBasedGlobalCompilationDatabase::setExtraFlagsForFile( - PathRef File, std::vector ExtraFlags) { - ExtraFlagsForFile[File] = std::move(ExtraFlags); -} - -tooling::CompilationDatabase * -DirectoryBasedGlobalCompilationDatabase::tryLoadDatabaseFromPath(PathRef File) { - - namespace path = llvm::sys::path; - auto CachedIt = CompilationDatabases.find(File); - - assert((path::is_absolute(File, path::Style::posix) || - path::is_absolute(File, path::Style::windows)) && - "path must be absolute"); - - if (CachedIt != CompilationDatabases.end()) - return CachedIt->second.get(); - std::string Error = ""; - auto CDB = tooling::CompilationDatabase::loadFromDirectory(File, Error); - if (CDB) { - auto Result = CDB.get(); - CompilationDatabases.insert(std::make_pair(File, std::move(CDB))); - return Result; +CompilationDatabase * +GlobalCompilationDatabase::getCompilationDatabase(PathRef File) const { + std::lock_guard Lock(Mutex); + std::string IgnoredErr; + for (auto Dir = path::parent_path(File); !Dir.empty(); + Dir = path::parent_path(Dir)) { + auto Insert = Cache.try_emplace(Dir, nullptr); + if (Insert.second) // We actually need to populate the cache. + Insert.first->getValue() = + CompilationDatabase::loadFromDirectory(Dir, IgnoredErr); + if (auto *DirDB = Insert.first->getValue().get()) + return DirDB; } - return nullptr; } -tooling::CompilationDatabase * -DirectoryBasedGlobalCompilationDatabase::getCompilationDatabase(PathRef File) { - std::lock_guard Lock(Mutex); - - namespace path = llvm::sys::path; - if (CompileCommandsDir.hasValue()) { - tooling::CompilationDatabase *ReturnValue = - tryLoadDatabaseFromPath(CompileCommandsDir.getValue()); - if (ReturnValue == nullptr) - Logger.log("Failed to find compilation database for " + Twine(File) + - "in overriden directory " + CompileCommandsDir.getValue() + - "\n"); - return ReturnValue; +std::vector +ExtraFlagsCompilationDatabase::getCompileCommands(PathRef File) const { + auto Commands = Inner->getCompileCommands(File); + std::vector Flags; + { + std::lock_guard Lock(Mutex); + auto It = ExtraFlags.find(File); + if (It == ExtraFlags.end()) + return Commands; + Flags = It->getValue(); } - - for (auto Path = path::parent_path(File); !Path.empty(); - Path = path::parent_path(Path)) { - auto CDB = tryLoadDatabaseFromPath(Path); - if (!CDB) - continue; - // FIXME(ibiryukov): Invalidate cached compilation databases on changes - return CDB; + for (auto &Cmd : Commands) { + assert(Cmd.CommandLine.size() >= 2 && + "Expected at least the command and the filename"); + auto It = Cmd.CommandLine.end(); + Cmd.CommandLine.insert(--It, make_move_iterator(Flags.begin()), + make_move_iterator(Flags.end())); } + return Commands; +} - Logger.log("Failed to find compilation database for " + Twine(File) + "\n"); - return nullptr; +void ExtraFlagsCompilationDatabase::setExtraFlagsForFile( + PathRef File, std::vector Flags) { + std::lock_guard Lock(Mutex); + ExtraFlags[File] = Flags; +} + +std::vector +FallbackCompilationDatabase::getCompileCommands(PathRef File) const { + auto Result = Primary->getCompileCommands(File); + if (Result.empty()) + Result = Fallback->getCompileCommands(File); + return Result; } } // namespace clangd Index: unittests/clangd/ClangdTests.cpp =================================================================== --- unittests/clangd/ClangdTests.cpp +++ unittests/clangd/ClangdTests.cpp @@ -203,7 +203,7 @@ VFSTag LastVFSTag = VFSTag(); }; -class MockCompilationDatabase : public GlobalCompilationDatabase { +class MockCompilationDatabase : public tooling::CompilationDatabase { public: MockCompilationDatabase(bool AddFreestandingFlag) { // We have to add -ffreestanding to VFS-specific tests to avoid errors on @@ -213,7 +213,7 @@ } std::vector - getCompileCommands(PathRef File) override { + getCompileCommands(PathRef File) const override { if (ExtraClangFlags.empty()) return {};