diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h --- a/clang-tools-extra/clangd/Preamble.h +++ b/clang-tools-extra/clangd/Preamble.h @@ -44,6 +44,30 @@ namespace clang { namespace clangd { +struct CapturedASTCtx { +public: + CapturedASTCtx(CompilerInstance &Clang) + : Invocation(Clang.getInvocationPtr()), + Diagnostics(Clang.getDiagnosticsPtr()), Target(Clang.getTargetPtr()), + AuxTarget(Clang.getAuxTarget()), FileMgr(Clang.getFileManagerPtr()), + SourceMgr(Clang.getSourceManagerPtr()), PP(Clang.getPreprocessorPtr()), + Context(Clang.getASTContextPtr()) {} + + ASTContext &getASTContext() { return *Context; } + Preprocessor &getPreprocessor() { return *PP; } + CompilerInvocation &getCompilerInvocation() { return *Invocation; } + +private: + std::shared_ptr Invocation; + IntrusiveRefCntPtr Diagnostics; + IntrusiveRefCntPtr Target; + IntrusiveRefCntPtr AuxTarget; + IntrusiveRefCntPtr FileMgr; + IntrusiveRefCntPtr SourceMgr; + std::shared_ptr PP; + IntrusiveRefCntPtr Context; +}; + /// The parsed preamble and associated data. /// /// As we must avoid re-parsing the preamble, any information that can only @@ -76,9 +100,6 @@ bool MainIsIncludeGuarded = false; }; -using PreambleParsedCallback = std::function; - /// Timings and statistics from the premble build. Unlike PreambleData, these /// do not need to be stored for later, but can be useful for logging, metrics, /// etc. @@ -101,10 +122,9 @@ /// If \p PreambleCallback is set, it will be run on top of the AST while /// building the preamble. /// If Stats is not non-null, build statistics will be exported there. -std::shared_ptr +std::pair, std::optional> buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, - PreambleParsedCallback PreambleCallback, PreambleBuildStats *Stats = nullptr); /// Returns true if \p Preamble is reusable for \p Inputs. Note that it will diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp --- a/clang-tools-extra/clangd/Preamble.cpp +++ b/clang-tools-extra/clangd/Preamble.cpp @@ -26,7 +26,9 @@ #include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" +#include "clang/Basic/TargetInfo.h" #include "clang/Basic/TokenKinds.h" +#include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/PrecompiledPreamble.h" @@ -35,6 +37,8 @@ #include "clang/Lex/PPCallbacks.h" #include "clang/Lex/Preprocessor.h" #include "clang/Lex/PreprocessorOptions.h" +#include "clang/Sema/Sema.h" +#include "clang/Serialization/ASTReader.h" #include "clang/Tooling/CompilationDatabase.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -77,10 +81,9 @@ class CppFilePreambleCallbacks : public PreambleCallbacks { public: CppFilePreambleCallbacks( - PathRef File, PreambleParsedCallback ParsedCallback, - PreambleBuildStats *Stats, bool ParseForwardingFunctions, + PathRef File, PreambleBuildStats *Stats, bool ParseForwardingFunctions, std::function BeforeExecuteCallback) - : File(File), ParsedCallback(ParsedCallback), Stats(Stats), + : File(File), Stats(Stats), ParseForwardingFunctions(ParseForwardingFunctions), BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {} @@ -95,13 +98,18 @@ } CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); } + CapturedASTCtx takeLife() { return std::move(*CapturedCtx); } + bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; } void AfterExecute(CompilerInstance &CI) override { - if (ParsedCallback) { - trace::Span Tracer("Running PreambleCallback"); - ParsedCallback(CI.getASTContext(), CI.getPreprocessor(), CanonIncludes); - } + CI.setSema(nullptr); + CI.setASTConsumer(nullptr); + CI.getASTReader()->setDeserializationListener(nullptr); + /* This just sets consumer to null when DeserializationListener is null */ + CI.getASTReader()->StartTranslationUnit(nullptr); + CI.getASTContext().setASTMutationListener(nullptr); + CapturedCtx.emplace(CI); const SourceManager &SM = CI.getSourceManager(); const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID()); @@ -202,7 +210,6 @@ private: PathRef File; - PreambleParsedCallback ParsedCallback; IncludeStructure Includes; CanonicalIncludes CanonIncludes; include_cleaner::PragmaIncludes Pragmas; @@ -216,6 +223,7 @@ PreambleBuildStats *Stats; bool ParseForwardingFunctions; std::function BeforeExecuteCallback; + std::optional CapturedCtx; }; // Represents directives other than includes, where basic textual information is @@ -577,16 +585,16 @@ }; } // namespace -std::shared_ptr +std::pair, std::optional> buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, - PreambleParsedCallback PreambleCallback, PreambleBuildStats *Stats) { // Note that we don't need to copy the input contents, preamble can live // without those. auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName); auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0); + std::optional CapturedCtx; trace::Span Tracer("BuildPreamble"); SPAN_ATTACH(Tracer, "File", FileName); @@ -635,8 +643,7 @@ CI.getPreprocessorOpts().WriteCommentListToPCH = false; CppFilePreambleCallbacks CapturedInfo( - FileName, PreambleCallback, Stats, - Inputs.Opts.PreambleParseForwardingFunctions, + FileName, Stats, Inputs.Opts.PreambleParseForwardingFunctions, [&ASTListeners](CompilerInstance &CI) { for (const auto &L : ASTListeners) L->beforeExecute(CI); @@ -682,7 +689,8 @@ Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes(); Result->StatCache = std::move(StatCache); Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded(); - return Result; + CapturedCtx.emplace(CapturedInfo.takeLife()); + return std::make_pair(Result, CapturedCtx); } elog("Could not build a preamble for file {0} version {1}: {2}", FileName, @@ -693,7 +701,8 @@ // Not an ideal way to show errors, but better than nothing! elog(" error: {0}", D.Message); } - return nullptr; + return std::make_pair, + std::optional>(nullptr, {}); } bool isPreambleCompatible(const PreambleData &Preamble, diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp --- a/clang-tools-extra/clangd/TUScheduler.cpp +++ b/clang-tools-extra/clangd/TUScheduler.cpp @@ -1049,16 +1049,27 @@ assert(Req.CI && "Got preamble request with null compiler invocation"); const ParseInputs &Inputs = Req.Inputs; bool ReusedPreamble = false; + std::optional CapturedCtx; Status.update([&](TUStatus &Status) { Status.PreambleActivity = PreambleAction::Building; }); - auto _ = llvm::make_scope_exit([this, &Req, &ReusedPreamble] { + auto _ = llvm::make_scope_exit([&] { ASTPeer.updatePreamble(std::move(Req.CI), std::move(Req.Inputs), LatestBuild, std::move(Req.CIDiags), std::move(Req.WantDiags)); - if (!ReusedPreamble) + if (!ReusedPreamble) { Callbacks.onPreamblePublished(FileName); + if (LatestBuild) { + assert(CapturedCtx); + CanonicalIncludes CanonIncludes = LatestBuild->CanonIncludes; + CompilerInvocation &CI = CapturedCtx->getCompilerInvocation(); + ASTContext &Ctx = CapturedCtx->getASTContext(); + Preprocessor &PP = CapturedCtx->getPreprocessor(); + Callbacks.onPreambleAST(FileName, Inputs.Version, CI, Ctx, PP, + CanonIncludes); + } + } }); if (!LatestBuild || Inputs.ForceRebuild) { @@ -1082,17 +1093,14 @@ PreambleBuildStats Stats; bool IsFirstPreamble = !LatestBuild; - LatestBuild = clang::clangd::buildPreamble( - FileName, *Req.CI, Inputs, StoreInMemory, - [&](ASTContext &Ctx, Preprocessor &PP, - const CanonicalIncludes &CanonIncludes) { - Callbacks.onPreambleAST(FileName, Inputs.Version, *Req.CI, Ctx, PP, - CanonIncludes); - }, - &Stats); - if (!LatestBuild) + auto BuildResult = clang::clangd::buildPreamble(FileName, *Req.CI, Inputs, + StoreInMemory, &Stats); + LatestBuild = BuildResult.first; + if (!LatestBuild) { return; + } reportPreambleBuild(Stats, IsFirstPreamble); + CapturedCtx = BuildResult.second; if (isReliable(LatestBuild->CompileCommand)) HeaderIncluders.update(FileName, LatestBuild->Includes.allHeaders()); } diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp --- a/clang-tools-extra/clangd/tool/Check.cpp +++ b/clang-tools-extra/clangd/tool/Check.cpp @@ -206,15 +206,9 @@ // Build preamble and AST, and index them. bool buildAST() { log("Building preamble..."); - Preamble = buildPreamble(File, *Invocation, Inputs, /*StoreInMemory=*/true, - [&](ASTContext &Ctx, Preprocessor &PP, - const CanonicalIncludes &Includes) { - if (!Opts.BuildDynamicSymbolIndex) - return; - log("Indexing headers..."); - Index.updatePreamble(File, /*Version=*/"null", - Ctx, PP, Includes); - }); + auto BuildResult = + buildPreamble(File, *Invocation, Inputs, /*StoreInMemory=*/true); + Preamble = BuildResult.first; if (!Preamble) { elog("Failed to build preamble"); return false; diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp --- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp +++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp @@ -133,7 +133,8 @@ return {}; } auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs, - /*InMemory=*/true, /*Callback=*/nullptr); + /*InMemory=*/true) + .first; return codeComplete(testPath(TU.Filename), Point, Preamble.get(), Inputs, Opts); } @@ -1323,7 +1324,8 @@ return {}; } auto Preamble = buildPreamble(testPath(TU.Filename), *CI, Inputs, - /*InMemory=*/true, /*Callback=*/nullptr); + /*InMemory=*/true) + .first; if (!Preamble) { ADD_FAILURE() << "Couldn't build Preamble"; return {}; @@ -1588,7 +1590,8 @@ auto CI = buildCompilerInvocation(Inputs, Diags); ASSERT_TRUE(CI); auto EmptyPreamble = buildPreamble(testPath(TU.Filename), *CI, Inputs, - /*InMemory=*/true, /*Callback=*/nullptr); + /*InMemory=*/true) + .first; ASSERT_TRUE(EmptyPreamble); TU.AdditionalFiles["a.h"] = "int foo(int x);"; diff --git a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp --- a/clang-tools-extra/clangd/unittests/FileIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/FileIndexTests.cpp @@ -305,16 +305,19 @@ FileIndex Index; bool IndexUpdated = false; - buildPreamble(FooCpp, *CI, PI, - /*StoreInMemory=*/true, - [&](ASTContext &Ctx, Preprocessor &PP, - const CanonicalIncludes &CanonIncludes) { - EXPECT_FALSE(IndexUpdated) - << "Expected only a single index update"; - IndexUpdated = true; - Index.updatePreamble(FooCpp, /*Version=*/"null", Ctx, PP, - CanonIncludes); - }); + auto res = buildPreamble(FooCpp, *CI, PI, + /*StoreInMemory=*/true); + auto CanonIncludes = res.first->CanonIncludes; + auto AST = *res.second; + auto index_fn = [&](CapturedASTCtx &CapturedCtx, + const CanonicalIncludes &CanonIncludes) { + EXPECT_FALSE(IndexUpdated) << "Expected only a single index update"; + IndexUpdated = true; + ASTContext &Ctx = CapturedCtx.getASTContext(); + Preprocessor &PP = CapturedCtx.getPreprocessor(); + Index.updatePreamble(FooCpp, /*Version=*/"null", Ctx, PP, CanonIncludes); + }; + index_fn(AST, CanonIncludes); ASSERT_TRUE(IndexUpdated); // Check the index contains symbols from the preamble, but not from the main diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp --- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp +++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp @@ -496,7 +496,7 @@ auto Inputs = TU.inputs(FS); auto CI = buildCompilerInvocation(Inputs, Diags); auto EmptyPreamble = - buildPreamble(testPath("foo.cpp"), *CI, Inputs, true, nullptr); + buildPreamble(testPath("foo.cpp"), *CI, Inputs, true).first; ASSERT_TRUE(EmptyPreamble); EXPECT_THAT(EmptyPreamble->Includes.MainFileIncludes, IsEmpty()); @@ -539,7 +539,7 @@ auto Inputs = TU.inputs(FS); auto CI = buildCompilerInvocation(Inputs, Diags); auto BaselinePreamble = - buildPreamble(testPath("foo.cpp"), *CI, Inputs, true, nullptr); + buildPreamble(testPath("foo.cpp"), *CI, Inputs, true).first; ASSERT_TRUE(BaselinePreamble); EXPECT_THAT(BaselinePreamble->Includes.MainFileIncludes, ElementsAre(testing::Field(&Inclusion::Written, ""))); diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp --- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp +++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp @@ -195,8 +195,10 @@ TU.AdditionalFiles["b.h"] = ""; TU.AdditionalFiles["c.h"] = ""; auto PI = TU.inputs(FS); - auto BaselinePreamble = buildPreamble( - TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true, nullptr); + auto BaselinePreamble = + buildPreamble(TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true, + nullptr) + .first; // We drop c.h from modified and add a new header. Since the latter is patched // we should only get a.h in preamble includes. d.h shouldn't be part of the // preamble, as it's coming from a disabled region. diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -1563,8 +1563,8 @@ CaptureBuiltFilenames(std::vector &Filenames) : Filenames(Filenames) {} void onPreambleAST(PathRef Path, llvm::StringRef Version, - const CompilerInvocation &CI, ASTContext &Ctx, - Preprocessor &PP, const CanonicalIncludes &) override { + const CompilerInvocation &, ASTContext &Ctx, + Preprocessor &, const CanonicalIncludes &) override { // Deliberately no synchronization. // The PreambleThrottler should serialize these calls, if not then tsan // will find a bug here. diff --git a/clang-tools-extra/clangd/unittests/TestTU.h b/clang-tools-extra/clangd/unittests/TestTU.h --- a/clang-tools-extra/clangd/unittests/TestTU.h +++ b/clang-tools-extra/clangd/unittests/TestTU.h @@ -32,6 +32,8 @@ namespace clang { namespace clangd { +using PreambleParsedCallback = std::function; struct TestTU { static TestTU withCode(llvm::StringRef Code) { TestTU TU; @@ -89,7 +91,7 @@ // The result will always have getDiagnostics() populated. ParsedAST build() const; std::shared_ptr - preamble(PreambleParsedCallback PreambleCallback = nullptr) const; + preamble(PreambleParsedCallback = nullptr) const; ParseInputs inputs(MockFS &FS) const; SymbolSlab headerSymbols() const; RefSlab headerRefs() const; diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp --- a/clang-tools-extra/clangd/unittests/TestTU.cpp +++ b/clang-tools-extra/clangd/unittests/TestTU.cpp @@ -107,8 +107,16 @@ initializeModuleCache(*CI); auto ModuleCacheDeleter = llvm::make_scope_exit( std::bind(deleteModuleCache, CI->getHeaderSearchOpts().ModuleCachePath)); - return clang::clangd::buildPreamble(testPath(Filename), *CI, Inputs, - /*StoreInMemory=*/true, PreambleCallback); + auto res = clang::clangd::buildPreamble(testPath(Filename), *CI, Inputs, + /*StoreInMemory=*/true); + if (PreambleCallback) { + auto &ASTCtx = *res.second; + ASTContext &Ctx = ASTCtx.getASTContext(); + Preprocessor &PP = ASTCtx.getPreprocessor(); + auto &CanonIncludes = res.first->CanonIncludes; + PreambleCallback(Ctx, PP, CanonIncludes); + } + return res.first; } ParsedAST TestTU::build() const { @@ -124,8 +132,8 @@ std::bind(deleteModuleCache, CI->getHeaderSearchOpts().ModuleCachePath)); auto Preamble = clang::clangd::buildPreamble(testPath(Filename), *CI, Inputs, - /*StoreInMemory=*/true, - /*PreambleCallback=*/nullptr); + /*StoreInMemory=*/true) + .first; auto AST = ParsedAST::build(testPath(Filename), Inputs, std::move(CI), Diags.take(), Preamble); if (!AST) { diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -12,6 +12,7 @@ #include "clang/AST/ASTConsumer.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/SourceManager.h" +#include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/PCHContainerOperations.h" #include "clang/Frontend/Utils.h" @@ -233,6 +234,8 @@ return *Invocation; } + std::shared_ptr getInvocationPtr() { return Invocation; } + /// setInvocation - Replace the current invocation. void setInvocation(std::shared_ptr Value); @@ -338,6 +341,11 @@ return *Diagnostics; } + IntrusiveRefCntPtr getDiagnosticsPtr() const { + assert(Diagnostics && "Compiler instance has no diagnostics!"); + return Diagnostics; + } + /// setDiagnostics - Replace the current diagnostics engine. void setDiagnostics(DiagnosticsEngine *Value); @@ -373,6 +381,11 @@ return *Target; } + IntrusiveRefCntPtr getTargetPtr() const { + assert(Target && "Compiler instance has no target!"); + return Target; + } + /// Replace the current Target. void setTarget(TargetInfo *Value); @@ -406,6 +419,11 @@ return *FileMgr; } + IntrusiveRefCntPtr getFileManagerPtr() const { + assert(FileMgr && "Compiler instance has no file manager!"); + return FileMgr; + } + void resetAndLeakFileManager() { llvm::BuryPointer(FileMgr.get()); FileMgr.resetWithoutRelease(); @@ -426,6 +444,11 @@ return *SourceMgr; } + IntrusiveRefCntPtr getSourceManagerPtr() const { + assert(SourceMgr && "Compiler instance has no source manager!"); + return SourceMgr; + } + void resetAndLeakSourceManager() { llvm::BuryPointer(SourceMgr.get()); SourceMgr.resetWithoutRelease(); @@ -466,6 +489,11 @@ return *Context; } + IntrusiveRefCntPtr getASTContextPtr() const { + assert(Context && "Compiler instance has no AST context!"); + return Context; + } + void resetAndLeakASTContext() { llvm::BuryPointer(Context.get()); Context.resetWithoutRelease();