Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -152,9 +152,11 @@ // environment to build the file, it would be nice if we could emit a // "PreparingBuild" status to inform users, it is non-trivial given the // current implementation. + log(" AddDocument with Index: {0}", Index); WorkScheduler.update(File, - ParseInputs{getCompileCommand(File), - FSProvider.getFileSystem(), Contents.str()}, + ParseInputs(getCompileCommand(File), + FSProvider.getFileSystem(), Contents.str(), + Index), WantDiags); } Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -16,6 +16,7 @@ #include "Headers.h" #include "Path.h" #include "Protocol.h" +#include "index/Index.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Frontend/PrecompiledPreamble.h" #include "clang/Lex/Preprocessor.h" @@ -62,9 +63,18 @@ /// Information required to run clang, e.g. to parse AST or do code completion. struct ParseInputs { + ParseInputs(tooling::CompileCommand CompileCommand, + IntrusiveRefCntPtr FS, + std::string Contents, const SymbolIndex *Index = nullptr) + : CompileCommand(CompileCommand), FS(std::move(FS)), + Contents(std::move(Contents)), Index(Index) {} + + ParseInputs() = default; + tooling::CompileCommand CompileCommand; IntrusiveRefCntPtr FS; std::string Contents; + const SymbolIndex *Index = nullptr; }; /// Stores and provides access to parsed AST. @@ -77,7 +87,8 @@ std::shared_ptr Preamble, std::unique_ptr Buffer, std::shared_ptr PCHs, - IntrusiveRefCntPtr VFS); + IntrusiveRefCntPtr VFS, + const SymbolIndex *Index); ParsedAST(ParsedAST &&Other); ParsedAST &operator=(ParsedAST &&Other); Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -10,13 +10,23 @@ #include "ClangdUnit.h" #include "../clang-tidy/ClangTidyDiagnosticConsumer.h" #include "../clang-tidy/ClangTidyModuleRegistry.h" +#include "AST.h" +#include "ClangdServer.h" #include "Compiler.h" #include "Diagnostics.h" +#include "Headers.h" #include "Logger.h" +#include "Protocol.h" #include "SourceCode.h" #include "Trace.h" +#include "index/Index.h" #include "clang/AST/ASTContext.h" +#include "clang/AST/DeclBase.h" +#include "clang/AST/DeclarationName.h" +#include "clang/AST/NestedNameSpecifier.h" +#include "clang/Basic/Diagnostic.h" #include "clang/Basic/LangOptions.h" +#include "clang/Basic/SourceLocation.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" @@ -27,14 +37,20 @@ #include "clang/Lex/MacroInfo.h" #include "clang/Lex/Preprocessor.h" #include "clang/Lex/PreprocessorOptions.h" +#include "clang/Sema/DeclSpec.h" +#include "clang/Sema/Lookup.h" #include "clang/Sema/Sema.h" #include "clang/Serialization/ASTWriter.h" #include "clang/Tooling/CompilationDatabase.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/raw_ostream.h" #include +#include namespace clang { namespace clangd { @@ -217,6 +233,260 @@ const LangOptions &LangOpts; }; +// TODO: merge with the copy in CodeComplete.cpp +static llvm::Expected toHeaderFile(llvm::StringRef Header, + llvm::StringRef HintPath) { + if (isLiteralInclude(Header)) + return HeaderFile{Header.str(), /*Verbatim=*/true}; + auto U = URI::parse(Header); + if (!U) + return U.takeError(); + + auto IncludePath = URI::includeSpelling(*U); + if (!IncludePath) + return IncludePath.takeError(); + if (!IncludePath->empty()) + return HeaderFile{std::move(*IncludePath), /*Verbatim=*/true}; + + auto Resolved = URI::resolve(*U, HintPath); + if (!Resolved) + return Resolved.takeError(); + return HeaderFile{std::move(*Resolved), /*Verbatim=*/false}; +} + +// TODO: add missing include for incomplete type e.g. only see forward decl. +// TODO: support the following case when clang-tidy header hasn't been included. +// namespace clang { +// void f() { +// tidy::Check c; +// ~~~~ +// // or +// clang::tidy::Check c; +// ~~~~ +// } +// } +// For both cases, the typo and the diagnostic are both on "tidy". However, +// the "typo" (or symbol) we want to fix is "clang::tidy::Check". +class IncludeFixer : public ExternalSemaSource, public DiagnosticConsumer { +public: + IncludeFixer(llvm::StringRef FileName, CompilerInstance *CI, + DiagnosticConsumer *Delegated, const IncludeInserter &Inserter, + const SymbolIndex *Index) + : FileName(FileName), Compiler(CI), Delegated(Delegated), + Inserter(Inserter), Index(Index) {} + + struct TypoRecord { + std::string Typo; + SourceLocation Loc; + Scope *S; + std::string SS; + bool IsValidSS; + DeclContext *MemberContext; + int LookupKind; + }; + + TypoCorrection CorrectTypo(const DeclarationNameInfo &Typo, int LookupKind, + Scope *S, CXXScopeSpec *SS, + CorrectionCandidateCallback &CCC, + DeclContext *MemberContext, bool EnteringContext, + const ObjCObjectPointerType *OPT) override { + log("IncludeFixer: CorrectTypo \"{0}\"", Typo.getAsString()); + auto &SemaRef = Compiler->getSema(); + if (SemaRef.isSFINAEContext()) + return TypoCorrection(); + const auto &SM = Compiler->getSourceManager(); + if (!SM.isWrittenInMainFile(Typo.getLoc())) + return clang::TypoCorrection(); + TypoRecord Record; + Record.Typo = Typo.getAsString(); + Record.Loc = Typo.getBeginLoc(); + Record.MemberContext = MemberContext; + Record.S = S; + Record.LookupKind = LookupKind; + + Record.SS = ""; + Record.IsValidSS = false; + if (SS) { + if (SS->isNotEmpty()) { + if (auto *Nested = SS->getScopeRep()) { + if (const auto *NS = Nested->getAsNamespace()) { + Record.SS = printNamespaceScope(*NS); + Record.IsValidSS = true; + } + } else { + auto QS = Lexer::getSourceText( + CharSourceRange::getCharRange(SS->getRange()), SM, + clang::LangOptions()) + .ltrim("::"); + Record.SS = QS; + } + } + } + LastTypo = std::move(Record); + + return TypoCorrection(); + } + + void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, + const clang::Diagnostic &Info) override { + if (Delegated) { + Delegated->HandleDiagnostic(DiagLevel, Info); + } + log("IncludeFixer::HandleDiagnostic (Index: {0})", Index); + if (DiagLevel >= DiagnosticsEngine::Error) { + log(">>>>>>>>>>>>>>>>>>>>>>> LogLevel >= Error!!!! <<<<<<<<<<<<"); + IgnoreDiagnostics::log(DiagLevel, Info); + if (LastTypo) { + if (LastTypo->Loc != Info.getLocation()) + return; + if (!Index) + return; + const auto &SM = Compiler->getSourceManager(); + auto *S = LastTypo->S; + log("========== Last typo ========= "); + if (LastTypo->MemberContext) + log(" MemberContext: {0}", + printNamespaceScope(*LastTypo->MemberContext)); + log(" TODO: handle typo: \"{0}\" at {1}", LastTypo->Typo, + LastTypo->Loc.printToString(SM)); + log(" CXXScopeSpec: \"{0}\" Valid: {1}", LastTypo->SS, + LastTypo->IsValidSS); + + std::vector Scopes; + if (LastTypo->SS.empty()) { + auto &SemaRef = Compiler->getSema(); + std::vector Visited; + VisitedContextCollector Collector(&Visited); + log(" Collecting contexts with lookup..."); + if (S) + SemaRef.LookupVisibleDecls( + S, (Sema::LookupNameKind)LastTypo->LookupKind, Collector, + /*IncludeGlobalScope=*/false, + /*LoadExternal=*/false); + log(" Collected {0} contexts.", Visited.size()); + Scopes.push_back(""); + for (const auto *Ctx : Visited) { + if (isa(Ctx)) + log(" global"); // global namespace + else if (isa(Ctx)) { + log(" {0}", printNamespaceScope(*Ctx)); + Scopes.push_back(printNamespaceScope(*Ctx)); + } + } + } else { + Scopes.push_back(LastTypo->SS); + } + log(" Collected scopes: [{0}]", + llvm::join(Scopes.begin(), Scopes.end(), ", ")); + std::string HeaderToInsert; + std::string DeclaringFile; + FuzzyFindRequest Req; + Req.AnyScope = false; + Req.Query = LastTypo->Typo; + Req.Scopes = Scopes; + Req.RestrictForCodeCompletion = true; + log(" ----- Querying index"); + Index->fuzzyFind(Req, [&](const Symbol &Sym) { + if (Sym.Name != Req.Query) + return; + if (!HeaderToInsert.empty()) + return; + log(" Symbol {0}{1} matched with {2} includes.", + Sym.Scope, Sym.Name, Sym.IncludeHeaders.size()); + if (!Sym.IncludeHeaders.empty()) { + HeaderToInsert = Sym.IncludeHeaders[0].IncludeHeader; + DeclaringFile = Sym.CanonicalDeclaration.FileURI; + } + }); + + if (HeaderToInsert.empty()) + return; + log(" ------>>>> Inserting new header: {0}", HeaderToInsert); + auto Inserted = [&](llvm::StringRef Header) + -> llvm::Expected> { + auto ResolvedDeclaring = toHeaderFile(DeclaringFile, FileName); + if (!ResolvedDeclaring) + return ResolvedDeclaring.takeError(); + auto ResolvedInserted = toHeaderFile(Header, FileName); + if (!ResolvedInserted) + return ResolvedInserted.takeError(); + return std::make_pair(Inserter.calculateIncludePath( + *ResolvedDeclaring, *ResolvedInserted), + Inserter.shouldInsertInclude( + *ResolvedDeclaring, *ResolvedInserted)); + }; + llvm::Optional Edit; + if (auto ToInclude = Inserted(HeaderToInsert)) { + CodeCompletion::IncludeCandidate Include; + Include.Header = ToInclude->first; + HeaderToInsert = ToInclude->first; + if (ToInclude->second) { + Edit = Inserter.insert(ToInclude->first); + } + } else + log("Failed to generate include insertion edits for adding " + "header " + "(FileURI='{0}', IncludeHeader='{1}') into {2}", + DeclaringFile, HeaderToInsert, FileName); + + if (!Edit) + return; + Diag D; + D.Range = diagnosticRange(Info, LangOptions()); + llvm::SmallString<64> Message; + Info.FormatDiagnostic(Message); + D.Message = ("Attempt to fix error: " + Message.str()).str(); + // D.Message = "Unknown identifier!!!!"; + D.InsideMainFile = true; + D.File = Info.getSourceManager().getFilename(Info.getLocation()); + D.Severity = DiagLevel; + D.Category = + DiagnosticIDs::getCategoryNameFromID( + DiagnosticIDs::getCategoryNumberForDiag(Info.getID())) + .str(); + llvm::SmallVector Edits; + Edits.push_back(std::move(*Edit)); + D.Fixes.push_back( + Fix{"Insert missing include " + + llvm::StringRef(HeaderToInsert).trim('<').trim('>').str(), + std::move(Edits)}); + log("Added new diagnostics for inserting {0}!!!!", HeaderToInsert); + Diags.push_back(std::move(D)); + LastTypo.reset(); // LastTypo has been handled. + } + } + } + + std::vector takeDiags() { return std::move(Diags); } + +private: + class VisitedContextCollector : public VisibleDeclConsumer { + public: + VisitedContextCollector(std::vector *Visited) + : Visited(Visited) {} + + void EnteredContext(DeclContext *Ctx) override { + Visited->push_back(Ctx); + } + + void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, DeclContext *Ctx, + bool InBaseClass) override {} + + private: + std::vector *Visited; + + }; + + std::string FileName; + CompilerInstance *Compiler; + DiagnosticConsumer *Delegated; + const IncludeInserter &Inserter; + const SymbolIndex *Index; + + std::vector Diags; + llvm::Optional LastTypo; +}; + } // namespace void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS) { @@ -228,7 +498,8 @@ std::shared_ptr Preamble, std::unique_ptr Buffer, std::shared_ptr PCHs, - llvm::IntrusiveRefCntPtr VFS) { + llvm::IntrusiveRefCntPtr VFS, + const SymbolIndex *Index) { assert(CI); // Command-line parsing sets DisableFree to true by default, but we don't want // to leak memory in clangd. @@ -237,9 +508,10 @@ Preamble ? &Preamble->Preamble : nullptr; StoreDiags ASTDiags; + std::string Content = Buffer->getBuffer(); auto Clang = prepareCompilerInstance(std::move(CI), PreamblePCH, std::move(Buffer), - std::move(PCHs), std::move(VFS), ASTDiags); + std::move(PCHs), VFS, ASTDiags); if (!Clang) return None; @@ -285,6 +557,38 @@ Check->registerMatchers(&CTFinder); } } + std::unique_ptr DiagClient; + DiagnosticConsumer *DiagClientPtr = nullptr; + auto &DiagEng =Clang->getDiagnostics(); + if (DiagEng.ownsClient()) { + DiagClient = DiagEng.takeClient(); + DiagClientPtr = DiagClient.get(); + } else { + DiagClientPtr = DiagEng.getClient(); + } + auto Style = format::getStyle(format::DefaultFormatStyle, MainInput.getFile(), + format::DefaultFallbackStyle, + Content, VFS.get()); + if (!Style) { + log("getStyle() failed for file {0}: {1}. Fallback is LLVM style.", + MainInput.getFile(), Style.takeError()); + Style = format::getLLVMStyle(); + } + auto CWD = VFS->getCurrentWorkingDirectory(); + if (auto Err = CWD.getError()) { + elog("Failed to get CWD from VFS: {0}", Err.message()); + } + IncludeInserter Inserter(MainInput.getFile(), Content, *Style, CWD.get(), + Clang->getPreprocessor().getHeaderSearchInfo()); + if (Preamble) { + for (const auto &Inc : Preamble->Includes.MainFileIncludes) + Inserter.addExisting(Inc); + } + IntrusiveRefCntPtr FixIncludes(new IncludeFixer( + MainInput.getFile(), Clang.get(), DiagClientPtr, Inserter, Index)); + DiagEng.setErrorLimit(0); + Clang->setExternalSemaSource(FixIncludes); + DiagEng.setClient(FixIncludes.get(), /*ShouldOwnClient=*/false); // Copy over the includes from the preamble, then combine with the // non-preamble includes below. @@ -302,6 +606,7 @@ log("Execute() failed when building AST for {0}", MainInput.getFile()); std::vector ParsedDecls = Action->takeTopLevelDecls(); + // AST traversals should exclude the preamble, to avoid performance cliffs. Clang->getASTContext().setTraversalScope(ParsedDecls); { @@ -313,6 +618,7 @@ // UnitDiagsConsumer is local, we can not store it in CompilerInstance that // has a longer lifetime. + //Clang->getDiagnostics().setClient(FixIncludes.get(), /*ShouldOwnClient=*/false); Clang->getDiagnostics().setClient(new IgnoreDiagnostics); // CompilerInstance won't run this callback, do it directly. ASTDiags.EndSourceFile(); @@ -325,6 +631,10 @@ // Add diagnostics from the preamble, if any. if (Preamble) Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end()); + auto IncludeFixerDiags = FixIncludes->takeDiags(); + if (!IncludeFixerDiags.empty()) + Diags.insert(Diags.begin(), IncludeFixerDiags.begin(), + IncludeFixerDiags.end()); return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action), std::move(ParsedDecls), std::move(Diags), std::move(Includes)); @@ -539,7 +849,7 @@ return ParsedAST::build(llvm::make_unique(*Invocation), Preamble, llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents), - PCHs, std::move(VFS)); + PCHs, std::move(VFS), Inputs.Index); } SourceLocation getBeginningOfIdentifier(ParsedAST &Unit, const Position &Pos, Index: clangd/Diagnostics.h =================================================================== --- clangd/Diagnostics.h +++ clangd/Diagnostics.h @@ -108,6 +108,8 @@ llvm::Optional LastDiag; }; +Range diagnosticRange(const clang::Diagnostic &D, const LangOptions &L); + } // namespace clangd } // namespace clang Index: clangd/Diagnostics.cpp =================================================================== --- clangd/Diagnostics.cpp +++ clangd/Diagnostics.cpp @@ -46,37 +46,6 @@ return L != R.getEnd() && M.isPointWithin(L, R.getBegin(), R.getEnd()); } -// Clang diags have a location (shown as ^) and 0 or more ranges (~~~~). -// LSP needs a single range. -Range diagnosticRange(const clang::Diagnostic &D, const LangOptions &L) { - auto &M = D.getSourceManager(); - auto Loc = M.getFileLoc(D.getLocation()); - for (const auto &CR : D.getRanges()) { - auto R = Lexer::makeFileCharRange(CR, M, L); - if (locationInRange(Loc, R, M)) - return halfOpenToRange(M, R); - } - llvm::Optional FallbackRange; - // The range may be given as a fixit hint instead. - for (const auto &F : D.getFixItHints()) { - auto R = Lexer::makeFileCharRange(F.RemoveRange, M, L); - if (locationInRange(Loc, R, M)) - return halfOpenToRange(M, R); - // If there's a fixit that performs insertion, it has zero-width. Therefore - // it can't contain the location of the diag, but it might be possible that - // this should be reported as range. For example missing semicolon. - if (R.getBegin() == R.getEnd() && Loc == R.getBegin()) - FallbackRange = halfOpenToRange(M, R); - } - if (FallbackRange) - return *FallbackRange; - // If no suitable range is found, just use the token at the location. - auto R = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(Loc), M, L); - if (!R.isValid()) // Fall back to location only, let the editor deal with it. - R = CharSourceRange::getCharRange(Loc); - return halfOpenToRange(M, R); -} - bool isInsideMainFile(const SourceLocation Loc, const SourceManager &M) { return Loc.isValid() && M.isWrittenInMainFile(M.getFileLoc(Loc)); } @@ -188,6 +157,38 @@ } } // namespace + +// Clang diags have a location (shown as ^) and 0 or more ranges (~~~~). +// LSP needs a single range. +Range diagnosticRange(const clang::Diagnostic &D, const LangOptions &L) { + auto &M = D.getSourceManager(); + auto Loc = M.getFileLoc(D.getLocation()); + for (const auto &CR : D.getRanges()) { + auto R = Lexer::makeFileCharRange(CR, M, L); + if (locationInRange(Loc, R, M)) + return halfOpenToRange(M, R); + } + llvm::Optional FallbackRange; + // The range may be given as a fixit hint instead. + for (const auto &F : D.getFixItHints()) { + auto R = Lexer::makeFileCharRange(F.RemoveRange, M, L); + if (locationInRange(Loc, R, M)) + return halfOpenToRange(M, R); + // If there's a fixit that performs insertion, it has zero-width. Therefore + // it can't contain the location of the diag, but it might be possible that + // this should be reported as range. For example missing semicolon. + if (R.getBegin() == R.getEnd() && Loc == R.getBegin()) + FallbackRange = halfOpenToRange(M, R); + } + if (FallbackRange) + return *FallbackRange; + // If no suitable range is found, just use the token at the location. + auto R = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(Loc), M, L); + if (!R.isValid()) // Fall back to location only, let the editor deal with it. + R = CharSourceRange::getCharRange(Loc); + return halfOpenToRange(M, R); +} + llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const DiagBase &D) { OS << "["; if (!D.InsideMainFile) Index: unittests/clangd/CodeCompleteTests.cpp =================================================================== --- unittests/clangd/CodeCompleteTests.cpp +++ unittests/clangd/CodeCompleteTests.cpp @@ -2320,6 +2320,26 @@ EXPECT_THAT(C, ElementsAre(SnippetSuffix("${1:(unsigned int)}"))); } +TEST(CompletionTest, CorrectTypo) { + auto Results = completions(R"cpp( + namespace nx { + namespace nz { + void f() { + nx::Clangd x; + ::Clangg y; + Clangc xx; + plus(Clangd, some()); + ^ + } + } + } + )cpp", + {cls("nx::Clangd1"), cls("nx::Clangd2")}); + // Although Clangd1 is from another namespace, Sema tells us it's in-scope and + // needs no qualifier. + EXPECT_THAT(Results.Completions, UnorderedElementsAre()); +} + } // namespace } // namespace clangd } // namespace clang Index: unittests/clangd/FileIndexTests.cpp =================================================================== --- unittests/clangd/FileIndexTests.cpp +++ unittests/clangd/FileIndexTests.cpp @@ -361,10 +361,10 @@ /*StoreInMemory=*/true, [&](ASTContext &Ctx, std::shared_ptr PP) {}); // Build AST for main file with preamble. - auto AST = - ParsedAST::build(createInvocationFromCommandLine(Cmd), PreambleData, - llvm::MemoryBuffer::getMemBufferCopy(Main.code()), - std::make_shared(), PI.FS); + auto AST = ParsedAST::build( + createInvocationFromCommandLine(Cmd), PreambleData, + llvm::MemoryBuffer::getMemBufferCopy(Main.code()), + std::make_shared(), PI.FS, nullptr); ASSERT_TRUE(AST); FileIndex Index; Index.updateMain(MainFile, *AST); Index: unittests/clangd/TUSchedulerTests.cpp =================================================================== --- unittests/clangd/TUSchedulerTests.cpp +++ unittests/clangd/TUSchedulerTests.cpp @@ -38,8 +38,8 @@ class TUSchedulerTests : public ::testing::Test { protected: ParseInputs getInputs(PathRef File, std::string Contents) { - return ParseInputs{*CDB.getCompileCommand(File), - buildTestFS(Files, Timestamps), std::move(Contents)}; + return ParseInputs(*CDB.getCompileCommand(File), + buildTestFS(Files, Timestamps), std::move(Contents)); } void updateWithCallback(TUScheduler &S, PathRef File,