Index: clang-tools-extra/trunk/clangd/ASTManager.h =================================================================== --- clang-tools-extra/trunk/clangd/ASTManager.h +++ clang-tools-extra/trunk/clangd/ASTManager.h @@ -0,0 +1,76 @@ +//===--- ASTManager.h - Clang AST manager -----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTMANAGER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTMANAGER_H + +#include "DocumentStore.h" +#include "JSONRPCDispatcher.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include +#include +#include + +namespace clang { +class ASTUnit; +class DiagnosticsEngine; +class PCHContainerOperations; +namespace tooling { +class CompilationDatabase; +} // namespace tooling + +namespace clangd { + +class ASTManager : public DocumentStoreListener { +public: + ASTManager(JSONOutput &Output, DocumentStore &Store); + ~ASTManager() override; + + void onDocumentAdd(StringRef Uri) override; + // FIXME: Implement onDocumentRemove + // FIXME: Implement codeComplete + +private: + JSONOutput &Output; + DocumentStore &Store; + + /// Loads a compilation database for URI. May return nullptr if it fails. The + /// database is cached for subsequent accesses. + clang::tooling::CompilationDatabase * + getOrCreateCompilationDatabaseForFile(StringRef Uri); + // Craetes a new ASTUnit for the document at Uri. + // FIXME: This calls chdir internally, which is thread unsafe. + std::unique_ptr + createASTUnitForFile(StringRef Uri, const DocumentStore &Docs); + + void runWorker(); + + /// Clang objects. + llvm::StringMap> ASTs; + llvm::StringMap> + CompilationDatabases; + std::shared_ptr PCHs; + + /// We run parsing on a separate thread. This thread looks into PendingRequest + /// as a 'one element work queue' as long as RequestIsPending is true. + std::thread ClangWorker; + /// Queue of requests. + std::deque RequestQueue; + /// Setting Done to true will make the worker thread terminate. + bool Done = false; + /// Condition variable to wake up the worker thread. + std::condition_variable ClangRequestCV; + /// Lock for accesses to RequestQueue and Done. + std::mutex RequestLock; +}; + +} // namespace clangd +} // namespace clang + +#endif Index: clang-tools-extra/trunk/clangd/ASTManager.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ASTManager.cpp +++ clang-tools-extra/trunk/clangd/ASTManager.cpp @@ -0,0 +1,201 @@ +//===--- ASTManager.cpp - Clang AST manager -------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ASTManager.h" +#include "JSONRPCDispatcher.h" +#include "Protocol.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/Path.h" +#include +#include +using namespace clang; +using namespace clangd; + +/// Retrieve a copy of the contents of every file in the store, for feeding into +/// ASTUnit. +static std::vector +getRemappedFiles(const DocumentStore &Docs) { + // FIXME: Use VFS instead. This would allow us to get rid of the chdir below. + std::vector RemappedFiles; + for (const auto &P : Docs.getAllDocuments()) { + StringRef FileName = P.first; + FileName.consume_front("file://"); + RemappedFiles.push_back(ASTUnit::RemappedFile( + FileName, + llvm::MemoryBuffer::getMemBufferCopy(P.second, FileName).release())); + } + return RemappedFiles; +} + +/// Convert from clang diagnostic level to LSP severity. +static int getSeverity(DiagnosticsEngine::Level L) { + switch (L) { + case DiagnosticsEngine::Remark: + return 4; + case DiagnosticsEngine::Note: + return 3; + case DiagnosticsEngine::Warning: + return 2; + case DiagnosticsEngine::Fatal: + case DiagnosticsEngine::Error: + return 1; + case DiagnosticsEngine::Ignored: + return 0; + } +} + +ASTManager::ASTManager(JSONOutput &Output, DocumentStore &Store) + : Output(Output), Store(Store), + PCHs(std::make_shared()), + ClangWorker([this]() { runWorker(); }) {} + +void ASTManager::runWorker() { + while (true) { + std::string File; + + { + std::unique_lock Lock(RequestLock); + // Check if there's another request pending. We keep parsing until + // our one-element queue is empty. + ClangRequestCV.wait(Lock, [this] { + return !RequestQueue.empty() || Done; + }); + + if (RequestQueue.empty() && Done) + return; + + File = std::move(RequestQueue.back()); + RequestQueue.pop_back(); + } // unlock. + + auto &Unit = ASTs[File]; // Only one thread can access this at a time. + + if (!Unit) { + Unit = createASTUnitForFile(File, this->Store); + } else { + // Do a reparse if this wasn't the first parse. + // FIXME: This might have the wrong working directory if it changed in the + // meantime. + Unit->Reparse(PCHs, getRemappedFiles(this->Store)); + } + + if (!Unit) + continue; + + // Send the diagnotics to the editor. + // FIXME: If the diagnostic comes from a different file, do we want to + // show them all? Right now we drop everything not coming from the + // main file. + // FIXME: Send FixIts to the editor. + std::string Diagnostics; + for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(), + DEnd = Unit->stored_diag_end(); + D != DEnd; ++D) { + if (!D->getLocation().isValid() || + !D->getLocation().getManager().isInMainFile(D->getLocation())) + continue; + Position P; + P.line = D->getLocation().getSpellingLineNumber() - 1; + P.character = D->getLocation().getSpellingColumnNumber(); + Range R = {P, P}; + Diagnostics += + R"({"range":)" + Range::unparse(R) + + R"(,"severity":)" + std::to_string(getSeverity(D->getLevel())) + + R"(,"message":")" + llvm::yaml::escape(D->getMessage()) + + R"("},)"; + } + + if (!Diagnostics.empty()) + Diagnostics.pop_back(); // Drop trailing comma. + Output.writeMessage( + R"({"jsonrpc":"2.0","method":"textDocument/publishDiagnostics","params":{"uri":")" + + File + R"(","diagnostics":[)" + Diagnostics + R"(]}})"); + } +} + +ASTManager::~ASTManager() { + { + std::lock_guard Guard(RequestLock); + // Wake up the clang worker thread, then exit. + Done = true; + ClangRequestCV.notify_one(); + } + ClangWorker.join(); +} + +void ASTManager::onDocumentAdd(StringRef Uri) { + std::lock_guard Guard(RequestLock); + // Currently we discard all pending requests and just enqueue the latest one. + RequestQueue.clear(); + RequestQueue.push_back(Uri); + ClangRequestCV.notify_one(); +} + +tooling::CompilationDatabase * +ASTManager::getOrCreateCompilationDatabaseForFile(StringRef Uri) { + auto &I = CompilationDatabases[Uri]; + if (I) + return I.get(); + + Uri.consume_front("file://"); + + std::string Error; + I = tooling::CompilationDatabase::autoDetectFromSource(Uri, Error); + Output.logs() << "Failed to load compilation database: " << Error << '\n'; + return I.get(); +} + +std::unique_ptr +ASTManager::createASTUnitForFile(StringRef Uri, const DocumentStore &Docs) { + tooling::CompilationDatabase *CDB = + getOrCreateCompilationDatabaseForFile(Uri); + + Uri.consume_front("file://"); + std::vector Commands; + + if (CDB) { + Commands = CDB->getCompileCommands(Uri); + // chdir. This is thread hostile. + if (!Commands.empty()) + llvm::sys::fs::set_current_path(Commands.front().Directory); + } + if (Commands.empty()) { + // Add a fake command line if we know nothing. + Commands.push_back(tooling::CompileCommand( + llvm::sys::path::parent_path(Uri), llvm::sys::path::filename(Uri), + {"clang", "-fsyntax-only", Uri.str()}, "")); + } + + // Inject the resource dir. + // FIXME: Don't overwrite it if it's already there. + static int Dummy; // Just an address in this process. + std::string ResourceDir = + CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy); + Commands.front().CommandLine.push_back("-resource-dir=" + ResourceDir); + + IntrusiveRefCntPtr Diags = + CompilerInstance::createDiagnostics(new DiagnosticOptions); + + std::vector ArgStrs; + for (const auto &S : Commands.front().CommandLine) + ArgStrs.push_back(S.c_str()); + + return std::unique_ptr(ASTUnit::LoadFromCommandLine( + &*ArgStrs.begin(), &*ArgStrs.end(), PCHs, Diags, ResourceDir, + /*OnlyLocalDecls=*/false, /*CaptureDiagnostics=*/true, + getRemappedFiles(Docs), + /*RemappedFilesKeepOriginalName=*/true, + /*PrecompilePreambleAfterNParses=*/1, /*TUKind=*/TU_Complete, + /*CacheCodeCompletionResults=*/true, + /*IncludeBriefCommentsInCodeCompletion=*/true, + /*AllowPCHWithCompilerErrors=*/true)); +} Index: clang-tools-extra/trunk/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/trunk/clangd/CMakeLists.txt +++ clang-tools-extra/trunk/clangd/CMakeLists.txt @@ -1,4 +1,5 @@ add_clang_executable(clangd + ASTManager.cpp ClangDMain.cpp JSONRPCDispatcher.cpp Protocol.cpp @@ -8,5 +9,7 @@ target_link_libraries(clangd clangBasic clangFormat + clangFrontend + clangTooling LLVMSupport ) Index: clang-tools-extra/trunk/clangd/ClangDMain.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangDMain.cpp +++ clang-tools-extra/trunk/clangd/ClangDMain.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "ASTManager.h" #include "DocumentStore.h" #include "JSONRPCDispatcher.h" #include "ProtocolHandlers.h" @@ -27,6 +28,8 @@ // Set up a document store and intialize all the method handlers for JSONRPC // dispatching. DocumentStore Store; + ASTManager AST(Out, Store); + Store.addListener(&AST); JSONRPCDispatcher Dispatcher(llvm::make_unique(Out)); Dispatcher.registerHandler("initialize", llvm::make_unique(Out)); Index: clang-tools-extra/trunk/clangd/DocumentStore.h =================================================================== --- clang-tools-extra/trunk/clangd/DocumentStore.h +++ clang-tools-extra/trunk/clangd/DocumentStore.h @@ -12,27 +12,65 @@ #include "clang/Basic/LLVM.h" #include "llvm/ADT/StringMap.h" +#include #include namespace clang { namespace clangd { +class DocumentStore; + +struct DocumentStoreListener { + virtual ~DocumentStoreListener() = default; + virtual void onDocumentAdd(StringRef Uri) {} + virtual void onDocumentRemove(StringRef Uri) {} +}; /// A container for files opened in a workspace, addressed by URI. The contents /// are owned by the DocumentStore. class DocumentStore { public: /// Add a document to the store. Overwrites existing contents. - void addDocument(StringRef Uri, StringRef Text) { Docs[Uri] = Text; } + void addDocument(StringRef Uri, StringRef Text) { + { + std::lock_guard Guard(DocsMutex); + Docs[Uri] = Text; + } + for (const auto &Listener : Listeners) + Listener->onDocumentAdd(Uri); + } /// Delete a document from the store. - void removeDocument(StringRef Uri) { Docs.erase(Uri); } + void removeDocument(StringRef Uri) { + { + std::lock_guard Guard(DocsMutex); + Docs.erase(Uri); + } + for (const auto &Listener : Listeners) + Listener->onDocumentRemove(Uri); + } /// Retrieve a document from the store. Empty string if it's unknown. - StringRef getDocument(StringRef Uri) const { - auto I = Docs.find(Uri); - return I == Docs.end() ? StringRef("") : StringRef(I->second); + std::string getDocument(StringRef Uri) const { + // FIXME: This could be a reader lock. + std::lock_guard Guard(DocsMutex); + return Docs.lookup(Uri); + } + + /// Add a listener. Does not take ownership. + void addListener(DocumentStoreListener *DSL) { Listeners.push_back(DSL); } + + /// Get name and constents of all documents in this store. + std::vector> getAllDocuments() const { + std::vector> AllDocs; + std::lock_guard Guard(DocsMutex); + for (const auto &P : Docs) + AllDocs.emplace_back(P.first(), P.second); + return AllDocs; } private: llvm::StringMap Docs; + std::vector Listeners; + + mutable std::mutex DocsMutex; }; } // namespace clangd Index: clang-tools-extra/trunk/test/clangd/diagnostics.test =================================================================== --- clang-tools-extra/trunk/test/clangd/diagnostics.test +++ clang-tools-extra/trunk/test/clangd/diagnostics.test @@ -0,0 +1,17 @@ +# RUN: clangd < %s | FileCheck %s +# It is absolutely vital that this file has CRLF line endings. +# +Content-Length: 125 + +{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}} +# +Content-Length: 152 + +{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///foo.c","languageId":"c","version":1,"text":"void main() {}"}}} +# +# CHECK: {"jsonrpc":"2.0","method":"textDocument/publishDiagnostics","params":{"uri":"file:///foo.c","diagnostics":[{"range":{"start": {"line": 0, "character": 1}, "end": {"line": 0, "character": 1}},"severity":2,"message":"return type of 'main' is not 'int'"},{"range":{"start": {"line": 0, "character": 1}, "end": {"line": 0, "character": 1}},"severity":3,"message":"change return type to 'int'"}]}} +# +# +Content-Length: 44 + +{"jsonrpc":"2.0","id":5,"method":"shutdown"}