Index: clang-tools-extra/trunk/clangd/ClangdServer.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdServer.cpp +++ clang-tools-extra/trunk/clangd/ClangdServer.cpp @@ -205,14 +205,11 @@ std::vector Result; auto TaggedFS = FSProvider.getTaggedFileSystem(File); - // It would be nice to use runOnUnitWithoutReparse here, but we can't - // guarantee the correctness of code completion cache here if we don't do the - // reparse. - Units.runOnUnit(File, *OverridenContents, ResourceDir, CDB, PCHs, - TaggedFS.Value, [&](ClangdUnit &Unit) { - Result = Unit.codeComplete(*OverridenContents, Pos, - TaggedFS.Value); - }); + Units.runOnUnitWithoutReparse(File, *OverridenContents, ResourceDir, CDB, + PCHs, TaggedFS.Value, [&](ClangdUnit &Unit) { + Result = Unit.codeComplete( + *OverridenContents, Pos, TaggedFS.Value); + }); return make_tagged(std::move(Result), TaggedFS.Tag); } Index: clang-tools-extra/trunk/clangd/ClangdUnit.h =================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnit.h +++ clang-tools-extra/trunk/clangd/ClangdUnit.h @@ -13,6 +13,9 @@ #include "Path.h" #include "Protocol.h" #include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/PrecompiledPreamble.h" +#include "clang/Serialization/ASTBitCodes.h" +#include "clang/Tooling/CompilationDatabase.h" #include "clang/Tooling/Core/Replacement.h" #include @@ -70,11 +73,82 @@ void dumpAST(llvm::raw_ostream &OS) const; private: + /// Stores and provides access to parsed AST. + class ParsedAST { + public: + /// Attempts to run Clang and store parsed AST. If \p Preamble is non-null + /// it is reused during parsing. + static llvm::Optional + Build(std::unique_ptr CI, + const PrecompiledPreamble *Preamble, + ArrayRef PreambleDeclIDs, + std::unique_ptr Buffer, + std::shared_ptr PCHs, + IntrusiveRefCntPtr VFS); + + ParsedAST(ParsedAST &&Other); + ParsedAST &operator=(ParsedAST &&Other); + + ~ParsedAST(); + + ASTContext &getASTContext(); + const ASTContext &getASTContext() const; + + Preprocessor &getPreprocessor(); + const Preprocessor &getPreprocessor() const; + + /// This function returns all top-level decls, including those that come + /// from Preamble. Decls, coming from Preamble, have to be deserialized, so + /// this call might be expensive. + ArrayRef getTopLevelDecls(); + + const std::vector &getDiagnostics() const; + + private: + ParsedAST(std::unique_ptr Clang, + std::unique_ptr Action, + std::vector TopLevelDecls, + std::vector PendingTopLevelDecls, + std::vector Diags); + + private: + void ensurePreambleDeclsDeserialized(); + + // We store an "incomplete" FrontendAction (i.e. no EndSourceFile was called + // on it) and CompilerInstance used to run it. That way we don't have to do + // complex memory management of all Clang structures on our own. (They are + // stored in CompilerInstance and cleaned up by + // FrontendAction.EndSourceFile). + std::unique_ptr Clang; + std::unique_ptr Action; + + // Data, stored after parsing. + std::vector Diags; + std::vector TopLevelDecls; + std::vector PendingTopLevelDecls; + }; + + // Store Preamble and all associated data + struct PreambleData { + PreambleData(PrecompiledPreamble Preamble, + std::vector TopLevelDeclIDs, + std::vector Diags); + + PrecompiledPreamble Preamble; + std::vector TopLevelDeclIDs; + std::vector Diags; + }; + + SourceLocation getBeginningOfIdentifier(const Position &Pos, + const FileEntry *FE) const; + Path FileName; - std::unique_ptr Unit; - std::shared_ptr PCHs; + tooling::CompileCommand Command; - SourceLocation getBeginningOfIdentifier(const Position& Pos, const FileEntry* FE) const; + llvm::Optional Preamble; + llvm::Optional Unit; + + std::shared_ptr PCHs; }; } // namespace clangd Index: clang-tools-extra/trunk/clangd/ClangdUnit.cpp =================================================================== --- clang-tools-extra/trunk/clangd/ClangdUnit.cpp +++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp @@ -9,16 +9,22 @@ #include "ClangdUnit.h" -#include "clang/Frontend/ASTUnit.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/Utils.h" -#include "clang/Index/IndexingAction.h" #include "clang/Index/IndexDataConsumer.h" +#include "clang/Index/IndexingAction.h" #include "clang/Lex/Lexer.h" #include "clang/Lex/MacroInfo.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "clang/Sema/Sema.h" +#include "clang/Serialization/ASTWriter.h" #include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/Format.h" #include @@ -26,6 +32,196 @@ using namespace clang::clangd; using namespace clang; +namespace { + +class DeclTrackingASTConsumer : public ASTConsumer { +public: + DeclTrackingASTConsumer(std::vector &TopLevelDecls) + : TopLevelDecls(TopLevelDecls) {} + + bool HandleTopLevelDecl(DeclGroupRef DG) override { + for (const Decl *D : DG) { + // ObjCMethodDecl are not actually top-level decls. + if (isa(D)) + continue; + + TopLevelDecls.push_back(D); + } + return true; + } + +private: + std::vector &TopLevelDecls; +}; + +class ClangdFrontendAction : public SyntaxOnlyAction { +public: + std::vector takeTopLevelDecls() { + return std::move(TopLevelDecls); + } + +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override { + return llvm::make_unique(/*ref*/ TopLevelDecls); + } + +private: + std::vector TopLevelDecls; +}; + +class ClangdUnitPreambleCallbacks : public PreambleCallbacks { +public: + std::vector takeTopLevelDeclIDs() { + return std::move(TopLevelDeclIDs); + } + + void AfterPCHEmitted(ASTWriter &Writer) override { + TopLevelDeclIDs.reserve(TopLevelDecls.size()); + for (Decl *D : TopLevelDecls) { + // Invalid top-level decls may not have been serialized. + if (D->isInvalidDecl()) + continue; + TopLevelDeclIDs.push_back(Writer.getDeclID(D)); + } + } + + void HandleTopLevelDecl(DeclGroupRef DG) override { + for (Decl *D : DG) { + if (isa(D)) + continue; + TopLevelDecls.push_back(D); + } + } + +private: + std::vector TopLevelDecls; + std::vector TopLevelDeclIDs; +}; + +/// Convert from clang diagnostic level to LSP severity. +static int getSeverity(DiagnosticsEngine::Level L) { + switch (L) { + case DiagnosticsEngine::Remark: + return 4; + case DiagnosticsEngine::Note: + return 3; + case DiagnosticsEngine::Warning: + return 2; + case DiagnosticsEngine::Fatal: + case DiagnosticsEngine::Error: + return 1; + case DiagnosticsEngine::Ignored: + return 0; + } + llvm_unreachable("Unknown diagnostic level!"); +} + +llvm::Optional toClangdDiag(StoredDiagnostic D) { + auto Location = D.getLocation(); + if (!Location.isValid() || !Location.getManager().isInMainFile(Location)) + return llvm::None; + + Position P; + P.line = Location.getSpellingLineNumber() - 1; + P.character = Location.getSpellingColumnNumber(); + Range R = {P, P}; + clangd::Diagnostic Diag = {R, getSeverity(D.getLevel()), D.getMessage()}; + + llvm::SmallVector FixItsForDiagnostic; + for (const FixItHint &Fix : D.getFixIts()) { + FixItsForDiagnostic.push_back(clang::tooling::Replacement( + Location.getManager(), Fix.RemoveRange, Fix.CodeToInsert)); + } + return DiagWithFixIts{Diag, std::move(FixItsForDiagnostic)}; +} + +class StoreDiagsConsumer : public DiagnosticConsumer { +public: + StoreDiagsConsumer(std::vector &Output) : Output(Output) {} + + void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, + const clang::Diagnostic &Info) override { + DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info); + + if (auto convertedDiag = toClangdDiag(StoredDiagnostic(DiagLevel, Info))) + Output.push_back(std::move(*convertedDiag)); + } + +private: + std::vector &Output; +}; + +class EmptyDiagsConsumer : public DiagnosticConsumer { +public: + void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, + const clang::Diagnostic &Info) override {} +}; + +std::unique_ptr +createCompilerInvocation(ArrayRef ArgList, + IntrusiveRefCntPtr Diags, + IntrusiveRefCntPtr VFS) { + auto CI = createInvocationFromCommandLine(ArgList, std::move(Diags), + std::move(VFS)); + // We rely on CompilerInstance to manage the resource (i.e. free them on + // EndSourceFile), but that won't happen if DisableFree is set to true. + // Since createInvocationFromCommandLine sets it to true, we have to override + // it. + CI->getFrontendOpts().DisableFree = false; + return CI; +} + +/// Creates a CompilerInstance from \p CI, with main buffer overriden to \p +/// Buffer and arguments to read the PCH from \p Preamble, if \p Preamble is not +/// null. Note that vfs::FileSystem inside returned instance may differ from \p +/// VFS if additional file remapping were set in command-line arguments. +/// On some errors, returns null. When non-null value is returned, it's expected +/// to be consumed by the FrontendAction as it will have a pointer to the \p +/// Buffer that will only be deleted if BeginSourceFile is called. +std::unique_ptr +prepareCompilerInstance(std::unique_ptr CI, + const PrecompiledPreamble *Preamble, + std::unique_ptr Buffer, + std::shared_ptr PCHs, + IntrusiveRefCntPtr VFS, + DiagnosticConsumer &DiagsClient) { + assert(VFS && "VFS is null"); + assert(!CI->getPreprocessorOpts().RetainRemappedFileBuffers && + "Setting RetainRemappedFileBuffers to true will cause a memory leak " + "of ContentsBuffer"); + + // NOTE: we use Buffer.get() when adding remapped files, so we have to make + // sure it will be released if no error is emitted. + if (Preamble) { + Preamble->AddImplicitPreamble(*CI, Buffer.get()); + } else { + CI->getPreprocessorOpts().addRemappedFile( + CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get()); + } + + auto Clang = llvm::make_unique(PCHs); + Clang->setInvocation(std::move(CI)); + Clang->createDiagnostics(&DiagsClient, false); + + if (auto VFSWithRemapping = createVFSFromCompilerInvocation( + Clang->getInvocation(), Clang->getDiagnostics(), VFS)) + VFS = VFSWithRemapping; + Clang->setVirtualFileSystem(VFS); + + Clang->setTarget(TargetInfo::CreateTargetInfo( + Clang->getDiagnostics(), Clang->getInvocation().TargetOpts)); + if (!Clang->hasTarget()) + return nullptr; + + // RemappedFileBuffers will handle the lifetime of the Buffer pointer, + // release it. + Buffer.release(); + return Clang; +} + +} // namespace + ClangdUnit::ClangdUnit(PathRef FileName, StringRef Contents, StringRef ResourceDir, std::shared_ptr PCHs, @@ -39,44 +235,54 @@ Commands.front().CommandLine.push_back("-resource-dir=" + std::string(ResourceDir)); - IntrusiveRefCntPtr Diags = - CompilerInstance::createDiagnostics(new DiagnosticOptions); - - std::vector ArgStrs; - for (const auto &S : Commands.front().CommandLine) - ArgStrs.push_back(S.c_str()); - - ASTUnit::RemappedFile RemappedSource( - FileName, - llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName).release()); - - auto ArgP = &*ArgStrs.begin(); - Unit = std::unique_ptr(ASTUnit::LoadFromCommandLine( - ArgP, ArgP + ArgStrs.size(), PCHs, Diags, ResourceDir, - /*OnlyLocalDecls=*/false, /*CaptureDiagnostics=*/true, RemappedSource, - /*RemappedFilesKeepOriginalName=*/true, - /*PrecompilePreambleAfterNParses=*/1, /*TUKind=*/TU_Prefix, - /*CacheCodeCompletionResults=*/true, - /*IncludeBriefCommentsInCodeCompletion=*/true, - /*AllowPCHWithCompilerErrors=*/true, - /*SkipFunctionBodies=*/false, - /*SingleFileParse=*/false, - /*UserFilesAreVolatile=*/false, /*ForSerialization=*/false, - /*ModuleFormat=*/llvm::None, - /*ErrAST=*/nullptr, VFS)); - assert(Unit && "Unit wasn't created"); + Command = std::move(Commands.front()); + reparse(Contents, VFS); } void ClangdUnit::reparse(StringRef Contents, IntrusiveRefCntPtr VFS) { - // Do a reparse if this wasn't the first parse. - // FIXME: This might have the wrong working directory if it changed in the - // meantime. - ASTUnit::RemappedFile RemappedSource( - FileName, - llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName).release()); + std::vector ArgStrs; + for (const auto &S : Command.CommandLine) + ArgStrs.push_back(S.c_str()); - Unit->Reparse(PCHs, RemappedSource, VFS); + std::unique_ptr CI; + { + // FIXME(ibiryukov): store diagnostics from CommandLine when we start + // reporting them. + EmptyDiagsConsumer CommandLineDiagsConsumer; + IntrusiveRefCntPtr CommandLineDiagsEngine = + CompilerInstance::createDiagnostics(new DiagnosticOptions, + &CommandLineDiagsConsumer, false); + CI = createCompilerInvocation(ArgStrs, CommandLineDiagsEngine, VFS); + } + assert(CI && "Couldn't create CompilerInvocation"); + + std::unique_ptr ContentsBuffer = + llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName); + + // Rebuild the preamble if it is missing or can not be reused. + auto Bounds = + ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0); + if (!Preamble || !Preamble->Preamble.CanReuse(*CI, ContentsBuffer.get(), + Bounds, VFS.get())) { + std::vector PreambleDiags; + StoreDiagsConsumer PreambleDiagnosticsConsumer(/*ref*/ PreambleDiags); + IntrusiveRefCntPtr PreambleDiagsEngine = + CompilerInstance::createDiagnostics( + &CI->getDiagnosticOpts(), &PreambleDiagnosticsConsumer, false); + ClangdUnitPreambleCallbacks SerializedDeclsCollector; + auto BuiltPreamble = PrecompiledPreamble::Build( + *CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs, + SerializedDeclsCollector); + if (BuiltPreamble) + Preamble = PreambleData(std::move(*BuiltPreamble), + SerializedDeclsCollector.takeTopLevelDeclIDs(), + std::move(PreambleDiags)); + } + Unit = ParsedAST::Build( + std::move(CI), Preamble ? &Preamble->Preamble : nullptr, + Preamble ? llvm::makeArrayRef(Preamble->TopLevelDeclIDs) : llvm::None, + std::move(ContentsBuffer), PCHs, VFS); } namespace { @@ -188,97 +394,164 @@ std::vector ClangdUnit::codeComplete(StringRef Contents, Position Pos, IntrusiveRefCntPtr VFS) { - CodeCompleteOptions CCO; - CCO.IncludeBriefComments = 1; - // This is where code completion stores dirty buffers. Need to free after - // completion. - SmallVector OwnedBuffers; - SmallVector StoredDiagnostics; - IntrusiveRefCntPtr DiagEngine( - new DiagnosticsEngine(new DiagnosticIDs, new DiagnosticOptions)); - std::vector Items; - CompletionItemsCollector Collector(&Items, CCO); + std::vector ArgStrs; + for (const auto &S : Command.CommandLine) + ArgStrs.push_back(S.c_str()); - ASTUnit::RemappedFile RemappedSource( - FileName, - llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName).release()); - - IntrusiveRefCntPtr FileMgr( - new FileManager(Unit->getFileSystemOpts(), VFS)); - IntrusiveRefCntPtr SourceMgr( - new SourceManager(*DiagEngine, *FileMgr)); - // CodeComplete seems to require fresh LangOptions. - LangOptions LangOpts = Unit->getLangOpts(); - // The language server protocol uses zero-based line and column numbers. - // The clang code completion uses one-based numbers. - Unit->CodeComplete(FileName, Pos.line + 1, Pos.character + 1, RemappedSource, - CCO.IncludeMacros, CCO.IncludeCodePatterns, - CCO.IncludeBriefComments, Collector, PCHs, *DiagEngine, - LangOpts, *SourceMgr, *FileMgr, StoredDiagnostics, - OwnedBuffers); - for (const llvm::MemoryBuffer *Buffer : OwnedBuffers) - delete Buffer; - return Items; -} + std::unique_ptr CI; + EmptyDiagsConsumer DummyDiagsConsumer; + { + IntrusiveRefCntPtr CommandLineDiagsEngine = + CompilerInstance::createDiagnostics(new DiagnosticOptions, + &DummyDiagsConsumer, false); + CI = createCompilerInvocation(ArgStrs, CommandLineDiagsEngine, VFS); + } + assert(CI && "Couldn't create CompilerInvocation"); + + std::unique_ptr ContentsBuffer = + llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName); + + // Attempt to reuse the PCH from precompiled preamble, if it was built. + const PrecompiledPreamble *PreambleForCompletion = nullptr; + if (Preamble) { + auto Bounds = + ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0); + if (Preamble->Preamble.CanReuse(*CI, ContentsBuffer.get(), Bounds, + VFS.get())) + PreambleForCompletion = &Preamble->Preamble; + } + + auto Clang = prepareCompilerInstance(std::move(CI), PreambleForCompletion, + std::move(ContentsBuffer), PCHs, VFS, + DummyDiagsConsumer); + auto &DiagOpts = Clang->getDiagnosticOpts(); + DiagOpts.IgnoreWarnings = true; + + auto &FrontendOpts = Clang->getFrontendOpts(); + FrontendOpts.SkipFunctionBodies = true; + + FrontendOpts.CodeCompleteOpts.IncludeGlobals = true; + // we don't handle code patterns properly yet, disable them. + FrontendOpts.CodeCompleteOpts.IncludeCodePatterns = false; + FrontendOpts.CodeCompleteOpts.IncludeMacros = true; + FrontendOpts.CodeCompleteOpts.IncludeBriefComments = true; + + FrontendOpts.CodeCompletionAt.FileName = FileName; + FrontendOpts.CodeCompletionAt.Line = Pos.line + 1; + FrontendOpts.CodeCompletionAt.Column = Pos.character + 1; -namespace { -/// Convert from clang diagnostic level to LSP severity. -static int getSeverity(DiagnosticsEngine::Level L) { - switch (L) { - case DiagnosticsEngine::Remark: - return 4; - case DiagnosticsEngine::Note: - return 3; - case DiagnosticsEngine::Warning: - return 2; - case DiagnosticsEngine::Fatal: - case DiagnosticsEngine::Error: - return 1; - case DiagnosticsEngine::Ignored: - return 0; + std::vector Items; + Clang->setCodeCompletionConsumer( + new CompletionItemsCollector(&Items, FrontendOpts.CodeCompleteOpts)); + + SyntaxOnlyAction Action; + if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) { + // FIXME(ibiryukov): log errors + return Items; } - llvm_unreachable("Unknown diagnostic level!"); + if (!Action.Execute()) { + // FIXME(ibiryukov): log errors + } + Action.EndSourceFile(); + + return Items; } -} // namespace std::vector ClangdUnit::getLocalDiagnostics() const { + if (!Unit) + return {}; // Parsing failed. + std::vector Result; - for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(), - DEnd = Unit->stored_diag_end(); - D != DEnd; ++D) { - if (!D->getLocation().isValid() || - !D->getLocation().getManager().isInMainFile(D->getLocation())) - continue; - Position P; - P.line = D->getLocation().getSpellingLineNumber() - 1; - P.character = D->getLocation().getSpellingColumnNumber(); - Range R = {P, P}; - clangd::Diagnostic Diag = {R, getSeverity(D->getLevel()), D->getMessage()}; - - llvm::SmallVector FixItsForDiagnostic; - for (const FixItHint &Fix : D->getFixIts()) { - FixItsForDiagnostic.push_back(clang::tooling::Replacement( - Unit->getSourceManager(), Fix.RemoveRange, Fix.CodeToInsert)); - } - Result.push_back({Diag, std::move(FixItsForDiagnostic)}); - } + auto PreambleDiagsSize = Preamble ? Preamble->Diags.size() : 0; + const auto &Diags = Unit->getDiagnostics(); + Result.reserve(PreambleDiagsSize + Diags.size()); + + if (Preamble) + Result.insert(Result.end(), Preamble->Diags.begin(), Preamble->Diags.end()); + Result.insert(Result.end(), Diags.begin(), Diags.end()); return Result; } void ClangdUnit::dumpAST(llvm::raw_ostream &OS) const { + if (!Unit) { + OS << ""; + return; // Parsing failed. + } Unit->getASTContext().getTranslationUnitDecl()->dump(OS, true); } +llvm::Optional +ClangdUnit::ParsedAST::Build(std::unique_ptr CI, + const PrecompiledPreamble *Preamble, + ArrayRef PreambleDeclIDs, + std::unique_ptr Buffer, + std::shared_ptr PCHs, + IntrusiveRefCntPtr VFS) { + + std::vector ASTDiags; + StoreDiagsConsumer UnitDiagsConsumer(/*ref*/ ASTDiags); + + auto Clang = + prepareCompilerInstance(std::move(CI), Preamble, std::move(Buffer), PCHs, + VFS, /*ref*/ UnitDiagsConsumer); + + // Recover resources if we crash before exiting this method. + llvm::CrashRecoveryContextCleanupRegistrar CICleanup( + Clang.get()); + + auto Action = llvm::make_unique(); + if (!Action->BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) { + // FIXME(ibiryukov): log error + return llvm::None; + } + if (!Action->Execute()) { + // FIXME(ibiryukov): log error + } + + // UnitDiagsConsumer is local, we can not store it in CompilerInstance that + // has a longer lifetime. + Clang->getDiagnostics().setClient(new EmptyDiagsConsumer); + + std::vector ParsedDecls = Action->takeTopLevelDecls(); + std::vector PendingDecls; + if (Preamble) { + PendingDecls.reserve(PreambleDeclIDs.size()); + PendingDecls.insert(PendingDecls.begin(), PreambleDeclIDs.begin(), + PreambleDeclIDs.end()); + } + + return ParsedAST(std::move(Clang), std::move(Action), std::move(ParsedDecls), + std::move(PendingDecls), std::move(ASTDiags)); +} + namespace { + +SourceLocation getMacroArgExpandedLocation(const SourceManager &Mgr, + const FileEntry *FE, + unsigned Offset) { + SourceLocation FileLoc = Mgr.translateFileLineCol(FE, 1, 1); + return Mgr.getMacroArgExpandedLocation(FileLoc.getLocWithOffset(Offset)); +} + +SourceLocation getMacroArgExpandedLocation(const SourceManager &Mgr, + const FileEntry *FE, Position Pos) { + SourceLocation InputLoc = + Mgr.translateFileLineCol(FE, Pos.line + 1, Pos.character + 1); + return Mgr.getMacroArgExpandedLocation(InputLoc); +} + /// Finds declarations locations that a given source location refers to. class DeclarationLocationsFinder : public index::IndexDataConsumer { std::vector DeclarationLocations; const SourceLocation &SearchedLocation; - ASTUnit &Unit; + const ASTContext &AST; + Preprocessor &PP; + public: DeclarationLocationsFinder(raw_ostream &OS, - const SourceLocation &SearchedLocation, ASTUnit &Unit) : - SearchedLocation(SearchedLocation), Unit(Unit) {} + const SourceLocation &SearchedLocation, + ASTContext &AST, Preprocessor &PP) + : SearchedLocation(SearchedLocation), AST(AST), PP(PP) {} std::vector takeLocations() { // Don't keep the same location multiple times. @@ -302,17 +575,17 @@ private: bool isSearchedLocation(FileID FID, unsigned Offset) const { - const SourceManager &SourceMgr = Unit.getSourceManager(); - return SourceMgr.getFileOffset(SearchedLocation) == Offset - && SourceMgr.getFileID(SearchedLocation) == FID; + const SourceManager &SourceMgr = AST.getSourceManager(); + return SourceMgr.getFileOffset(SearchedLocation) == Offset && + SourceMgr.getFileID(SearchedLocation) == FID; } - void addDeclarationLocation(const SourceRange& ValSourceRange) { - const SourceManager& SourceMgr = Unit.getSourceManager(); - const LangOptions& LangOpts = Unit.getLangOpts(); + void addDeclarationLocation(const SourceRange &ValSourceRange) { + const SourceManager &SourceMgr = AST.getSourceManager(); + const LangOptions &LangOpts = AST.getLangOpts(); SourceLocation LocStart = ValSourceRange.getBegin(); SourceLocation LocEnd = Lexer::getLocForEndOfToken(ValSourceRange.getEnd(), - 0, SourceMgr, LangOpts); + 0, SourceMgr, LangOpts); Position Begin; Begin.line = SourceMgr.getSpellingLineNumber(LocStart) - 1; Begin.character = SourceMgr.getSpellingColumnNumber(LocStart) - 1; @@ -330,24 +603,24 @@ void finish() override { // Also handle possible macro at the searched location. Token Result; - if (!Lexer::getRawToken(SearchedLocation, Result, Unit.getSourceManager(), - Unit.getASTContext().getLangOpts(), false)) { + if (!Lexer::getRawToken(SearchedLocation, Result, AST.getSourceManager(), + AST.getLangOpts(), false)) { if (Result.is(tok::raw_identifier)) { - Unit.getPreprocessor().LookUpIdentifierInfo(Result); + PP.LookUpIdentifierInfo(Result); } - IdentifierInfo* IdentifierInfo = Result.getIdentifierInfo(); + IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo(); if (IdentifierInfo && IdentifierInfo->hadMacroDefinition()) { std::pair DecLoc = - Unit.getSourceManager().getDecomposedExpansionLoc(SearchedLocation); + AST.getSourceManager().getDecomposedExpansionLoc(SearchedLocation); // Get the definition just before the searched location so that a macro // referenced in a '#undef MACRO' can still be found. - SourceLocation BeforeSearchedLocation = Unit.getLocation( - Unit.getSourceManager().getFileEntryForID(DecLoc.first), + SourceLocation BeforeSearchedLocation = getMacroArgExpandedLocation( + AST.getSourceManager(), + AST.getSourceManager().getFileEntryForID(DecLoc.first), DecLoc.second - 1); MacroDefinition MacroDef = - Unit.getPreprocessor().getMacroDefinitionAtLoc(IdentifierInfo, - BeforeSearchedLocation); - MacroInfo* MacroInf = MacroDef.getMacroInfo(); + PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation); + MacroInfo *MacroInf = MacroDef.getMacroInfo(); if (MacroInf) { addDeclarationLocation( SourceRange(MacroInf->getDefinitionLoc(), @@ -360,30 +633,41 @@ } // namespace std::vector ClangdUnit::findDefinitions(Position Pos) { - const FileEntry *FE = Unit->getFileManager().getFile(Unit->getMainFileName()); + if (!Unit) + return {}; // Parsing failed. + + const SourceManager &SourceMgr = Unit->getASTContext().getSourceManager(); + const FileEntry *FE = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()); if (!FE) return {}; SourceLocation SourceLocationBeg = getBeginningOfIdentifier(Pos, FE); auto DeclLocationsFinder = std::make_shared( - llvm::errs(), SourceLocationBeg, *Unit); + llvm::errs(), SourceLocationBeg, Unit->getASTContext(), + Unit->getPreprocessor()); index::IndexingOptions IndexOpts; IndexOpts.SystemSymbolFilter = index::IndexingOptions::SystemSymbolFilterKind::All; IndexOpts.IndexFunctionLocals = true; - index::indexASTUnit(*Unit, DeclLocationsFinder, IndexOpts); + + indexTopLevelDecls(Unit->getASTContext(), Unit->getTopLevelDecls(), + DeclLocationsFinder, IndexOpts); return DeclLocationsFinder->takeLocations(); } SourceLocation ClangdUnit::getBeginningOfIdentifier(const Position &Pos, - const FileEntry *FE) const { + const FileEntry *FE) const { + // The language server protocol uses zero-based line and column numbers. // Clang uses one-based numbers. - SourceLocation InputLocation = Unit->getLocation(FE, Pos.line + 1, - Pos.character + 1); + const ASTContext &AST = Unit->getASTContext(); + const SourceManager &SourceMgr = AST.getSourceManager(); + + SourceLocation InputLocation = + getMacroArgExpandedLocation(SourceMgr, FE, Pos); if (Pos.character == 0) { return InputLocation; } @@ -396,20 +680,97 @@ // token. If so, Take the beginning of this token. // (It should be the same identifier because you can't have two adjacent // identifiers without another token in between.) - SourceLocation PeekBeforeLocation = Unit->getLocation(FE, Pos.line + 1, - Pos.character); - const SourceManager &SourceMgr = Unit->getSourceManager(); + SourceLocation PeekBeforeLocation = getMacroArgExpandedLocation( + SourceMgr, FE, Position{Pos.line, Pos.character - 1}); Token Result; if (Lexer::getRawToken(PeekBeforeLocation, Result, SourceMgr, - Unit->getASTContext().getLangOpts(), false)) { + AST.getLangOpts(), false)) { // getRawToken failed, just use InputLocation. return InputLocation; } if (Result.is(tok::raw_identifier)) { return Lexer::GetBeginningOfToken(PeekBeforeLocation, SourceMgr, - Unit->getASTContext().getLangOpts()); + Unit->getASTContext().getLangOpts()); } return InputLocation; } + +void ClangdUnit::ParsedAST::ensurePreambleDeclsDeserialized() { + if (PendingTopLevelDecls.empty()) + return; + + std::vector Resolved; + Resolved.reserve(PendingTopLevelDecls.size()); + + ExternalASTSource &Source = *getASTContext().getExternalSource(); + for (serialization::DeclID TopLevelDecl : PendingTopLevelDecls) { + // Resolve the declaration ID to an actual declaration, possibly + // deserializing the declaration in the process. + if (Decl *D = Source.GetExternalDecl(TopLevelDecl)) + Resolved.push_back(D); + } + + TopLevelDecls.reserve(TopLevelDecls.size() + PendingTopLevelDecls.size()); + TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end()); + + PendingTopLevelDecls.clear(); +} + +ClangdUnit::ParsedAST::ParsedAST(ParsedAST &&Other) = default; + +ClangdUnit::ParsedAST &ClangdUnit::ParsedAST:: +operator=(ParsedAST &&Other) = default; + +ClangdUnit::ParsedAST::~ParsedAST() { + if (Action) { + Action->EndSourceFile(); + } +} + +ASTContext &ClangdUnit::ParsedAST::getASTContext() { + return Clang->getASTContext(); +} + +const ASTContext &ClangdUnit::ParsedAST::getASTContext() const { + return Clang->getASTContext(); +} + +Preprocessor &ClangdUnit::ParsedAST::getPreprocessor() { + return Clang->getPreprocessor(); +} + +const Preprocessor &ClangdUnit::ParsedAST::getPreprocessor() const { + return Clang->getPreprocessor(); +} + +ArrayRef ClangdUnit::ParsedAST::getTopLevelDecls() { + ensurePreambleDeclsDeserialized(); + return TopLevelDecls; +} + +const std::vector & +ClangdUnit::ParsedAST::getDiagnostics() const { + return Diags; +} + +ClangdUnit::ParsedAST::ParsedAST( + std::unique_ptr Clang, + std::unique_ptr Action, + std::vector TopLevelDecls, + std::vector PendingTopLevelDecls, + std::vector Diags) + : Clang(std::move(Clang)), Action(std::move(Action)), + Diags(std::move(Diags)), TopLevelDecls(std::move(TopLevelDecls)), + PendingTopLevelDecls(std::move(PendingTopLevelDecls)) { + assert(this->Clang); + assert(this->Action); +} + +ClangdUnit::PreambleData::PreambleData( + PrecompiledPreamble Preamble, + std::vector TopLevelDeclIDs, + std::vector Diags) + : Preamble(std::move(Preamble)), + TopLevelDeclIDs(std::move(TopLevelDeclIDs)), Diags(std::move(Diags)) {}