[clangd] Unify compiler invocation creation

[kadircet]

Summary

Background-indexing is fine, because it uses GlobalCompilationDatabase
to fetch the compile commands (hence uses CommandMangler), and creates
invocation through buildCompilerInvocation.

Depends on D106639.

[sammccall]

clang-tools-extra/clangd/Compiler.h

64

Comment is good, but call this disableUnsupportedOptions?
(We're in namespace clangd already)

64

This should also probably have a comment about being paired with CommandMangler, as that provides similar functionality that just happens to be best expressed on the command instead (e.g. stripping --save-temps)

clang-tools-extra/clangd/test/indexer.test

4

Add a comment (or rename test file) saying what functionality this is trying to test? it's not obvious

[kadircet]

• Address comments

clang-tools-extra/clangd/CompileCommands.h

 namespace clangd {
 
 // CommandMangler transforms compile commands from some external source
 // for use in clangd. This means:
 //  - running the frontend only, stripping args regarding output files etc
 //  - forcing the use of clangd's builtin headers rather than clang's
 //  - resolving argv0 as cc1 expects
 //  - injecting -isysroot flags on mac as the system clang does
-// FIXME: This is currently not used in all code paths that create invocations.
-// Make use of these adjusters and buildCompilerInvocation in clangd-indexer as
-// well. It should be possible to hook it up by overriding RunInvocation in
-// FrontendActionFactory.
 struct CommandMangler {
   // Absolute path to clang.
   llvm::Optional<std::string> ClangPath;
   // Directory containing builtin headers.
   llvm::Optional<std::string> ResourceDir;
   // Root for searching for standard library (passed to -isysroot).
   llvm::Optional<std::string> Sysroot;
 

clang-tools-extra/clangd/Compiler.h

struct ParseInputs {
   // Used to recover from diagnostics (e.g. find missing includes for symbol).
   const SymbolIndex *Index = nullptr;
   ParseOptions Opts = ParseOptions();
   TidyProviderRef ClangTidyProvider = {};
   // Used to acquire ASTListeners when parsing files.
   FeatureModuleSet *FeatureModules = nullptr;
 };
 
+/// Clears \p CI from options that are not supported by clangd, like codegen or
+/// plugins. This should be combined with CommandMangler::adjust, which provides
+/// similar functionality for options that needs to be stripped from compile
+/// flags.
+void disableUnsupportedOptions(CompilerInvocation &CI);
+
 /// Builds compiler invocation that could be used to build AST or preamble.
 std::unique_ptr<CompilerInvocation>
 buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D,
                         std::vector<std::string> *CC1Args = nullptr);
 
 /// Creates a compiler instance, configured so that:
 ///   - Contents of the parsed file are remapped to \p MainFile.
 ///   - Preamble is overriden to use PCH passed to this function. It means the

clang-tools-extra/clangd/Compiler.cpp

 //===--- Compiler.cpp --------------------------------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Compiler.h"
 #include "support/Logger.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Serialization/PCHContainerOperations.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormatVariadic.h"
 
 namespace clang {
 namespace clangd {
void IgnoreDiagnostics::log(DiagnosticsEngine::Level DiagLevel,
   clangd::vlog("Ignored diagnostic. {0}{1}", Location, Message);
 }
 
 void IgnoreDiagnostics::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                                          const clang::Diagnostic &Info) {
   IgnoreDiagnostics::log(DiagLevel, Info);
 }
 
+void disableUnsupportedOptions(CompilerInvocation &CI) {
+  // Disable "clang -verify" diagnostics, they are rarely useful in clangd, and
+  // our compiler invocation set-up doesn't seem to work with it (leading
+  // assertions in VerifyDiagnosticConsumer).
+  CI.getDiagnosticOpts().VerifyDiagnostics = false;
+  CI.getDiagnosticOpts().ShowColors = false;
+
+  // Disable any dependency outputting, we don't want to generate files or write
+  // to stdout/stderr.
+  CI.getDependencyOutputOpts().ShowIncludesDest = ShowIncludesDestination::None;
+  CI.getDependencyOutputOpts().OutputFile.clear();
+  CI.getDependencyOutputOpts().HeaderIncludeOutputFile.clear();
+  CI.getDependencyOutputOpts().DOTOutputFile.clear();
+  CI.getDependencyOutputOpts().ModuleDependencyOutputDir.clear();
+
+  // Disable any pch generation/usage operations. Since serialized preamble
+  // format is unstable, using an incompatible one might result in unexpected
+  // behaviours, including crashes.
+  CI.getPreprocessorOpts().ImplicitPCHInclude.clear();
+  CI.getPreprocessorOpts().PrecompiledPreambleBytes = {0, false};
+  CI.getPreprocessorOpts().PCHThroughHeader.clear();
+  CI.getPreprocessorOpts().PCHWithHdrStop = false;
+  CI.getPreprocessorOpts().PCHWithHdrStopCreate = false;
+  // Don't crash on `#pragma clang __debug parser_crash`
+  CI.getPreprocessorOpts().DisablePragmaDebugCrash = true;
+
+  // Always default to raw container format as clangd doesn't registry any other
+  // and clang dies when faced with unknown formats.
+  CI.getHeaderSearchOpts().ModuleFormat =
+      PCHContainerOperations().getRawReader().getFormat().str();
+
+  CI.getFrontendOpts().Plugins.clear();
+  CI.getFrontendOpts().AddPluginActions.clear();
+  CI.getFrontendOpts().PluginArgs.clear();
+  CI.getFrontendOpts().ProgramAction = frontend::ParseSyntaxOnly;
+  CI.getFrontendOpts().ActionName.clear();
+}
+
 std::unique_ptr<CompilerInvocation>
 buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D,
                         std::vector<std::string> *CC1Args) {
   std::vector<const char *> ArgStrs;
   for (const auto &S : Inputs.CompileCommand.CommandLine)
     ArgStrs.push_back(S.c_str());
 
   auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
   llvm::IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
       CompilerInstance::createDiagnostics(new DiagnosticOptions, &D, false);
   std::unique_ptr<CompilerInvocation> CI = createInvocationFromCommandLine(
       ArgStrs, CommandLineDiagsEngine, std::move(VFS),
       /*ShouldRecoverOnErrors=*/true, CC1Args);
   if (!CI)
     return nullptr;
   // createInvocationFromCommandLine sets DisableFree.
   CI->getFrontendOpts().DisableFree = false;
   CI->getLangOpts()->CommentOpts.ParseAllComments = true;
   CI->getLangOpts()->RetainCommentsFromSystemHeaders = true;
-  // Disable "clang -verify" diagnostics, they are rarely useful in clangd, and
-  // our compiler invocation set-up doesn't seem to work with it (leading
-  // assertions in VerifyDiagnosticConsumer).
-  CI->getDiagnosticOpts().VerifyDiagnostics = false;
-  CI->getDiagnosticOpts().ShowColors = false;
-
-  // Disable any dependency outputting, we don't want to generate files or write
-  // to stdout/stderr.
-  CI->getDependencyOutputOpts().ShowIncludesDest =
-      ShowIncludesDestination::None;
-  CI->getDependencyOutputOpts().OutputFile.clear();
-  CI->getDependencyOutputOpts().HeaderIncludeOutputFile.clear();
-  CI->getDependencyOutputOpts().DOTOutputFile.clear();
-  CI->getDependencyOutputOpts().ModuleDependencyOutputDir.clear();
-
-  // Disable any pch generation/usage operations. Since serialized preamble
-  // format is unstable, using an incompatible one might result in unexpected
-  // behaviours, including crashes.
-  CI->getPreprocessorOpts().ImplicitPCHInclude.clear();
-  CI->getPreprocessorOpts().PrecompiledPreambleBytes = {0, false};
-  CI->getPreprocessorOpts().PCHThroughHeader.clear();
-  CI->getPreprocessorOpts().PCHWithHdrStop = false;
-  CI->getPreprocessorOpts().PCHWithHdrStopCreate = false;
-  // Don't crash on `#pragma clang __debug parser_crash`
-  CI->getPreprocessorOpts().DisablePragmaDebugCrash = true;
-
-  // Always default to raw container format as clangd doesn't registry any other
-  // and clang dies when faced with unknown formats.
-  CI->getHeaderSearchOpts().ModuleFormat =
-      PCHContainerOperations().getRawReader().getFormat().str();
-
-  CI->getFrontendOpts().Plugins.clear();
-  CI->getFrontendOpts().AddPluginActions.clear();
-  CI->getFrontendOpts().PluginArgs.clear();
-  CI->getFrontendOpts().ProgramAction = frontend::ParseSyntaxOnly;
-  CI->getFrontendOpts().ActionName.clear();
 
+  disableUnsupportedOptions(*CI);
   return CI;
 }
 
 std::unique_ptr<CompilerInstance>
 prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI,
                         const PrecompiledPreamble *Preamble,
                         std::unique_ptr<llvm::MemoryBuffer> Buffer,
                         llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,

clang-tools-extra/clangd/indexer/IndexerMain.cpp

 //===--- IndexerMain.cpp -----------------------------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // clangd-indexer is a tool to gather index data (symbols, xrefs) from source.
 //
 //===----------------------------------------------------------------------===//
 
+#include "CompileCommands.h"
+#include "Compiler.h"
 #include "index/IndexAction.h"
 #include "index/Merge.h"
 #include "index/Ref.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolCollector.h"
 #include "support/Logger.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Execution.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Signals.h"
+#include <utility>
 
 namespace clang {
 namespace clangd {
 namespace {
 
 static llvm::cl::opt<IndexFileFormat>
     Format("format", llvm::cl::desc("Format of the index to be written"),
            llvm::cl::values(clEnumValN(IndexFileFormat::YAML, "yaml",
return createStaticIndexingAction(
           std::lock_guard<std::mutex> Lock(RelsMu);
           for (const auto &R : S) {
             Relations.insert(R);
           }
         },
         /*IncludeGraphCallback=*/nullptr);
   }
 
+  bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
+                     FileManager *Files,
+                     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+                     DiagnosticConsumer *DiagConsumer) override {
+    disableUnsupportedOptions(*Invocation);
+    return tooling::FrontendActionFactory::runInvocation(
+        std::move(Invocation), Files, std::move(PCHContainerOps), DiagConsumer);
+  }
+
   // Awkward: we write the result in the destructor, because the executor
   // takes ownership so it's the easiest way to get our data back out.
   ~IndexActionFactory() {
     Result.Symbols = std::move(Symbols).build();
     Result.Refs = std::move(Refs).build();
     Result.Relations = std::move(Relations).build();
   }
 
if (!Executor) {
     llvm::errs() << llvm::toString(Executor.takeError()) << "\n";
     return 1;
   }
 
   // Collect symbols found in each translation unit, merging as we go.
   clang::clangd::IndexFileIn Data;
   auto Err = Executor->get()->execute(
       std::make_unique<clang::clangd::IndexActionFactory>(Data),
-      clang::tooling::getStripPluginsAdjuster());
+      clang::tooling::ArgumentsAdjuster(
+          clang::clangd::CommandMangler::detect()));
   if (Err) {
     clang::clangd::elog("{0}", std::move(Err));
   }
 
   // Emit collected data.
   clang::clangd::IndexFileOut Out(Data);
   Out.Format = clang::clangd::Format;
   llvm::outs() << Out;
   return 0;
 }

clang-tools-extra/clangd/test/indexer.test

+# RUN: rm -rf %t.cpp
+# RUN: touch %t.cpp
+#
+# Make sure compile flags are adjusted for clangd. `--save-temps` creates a
+# `.ii` file and `-verify` triggers extra diagnostics generation. Clangd should
+# strip those.
+# RUN: clangd-indexer %t.cpp -- -Xclang -verify --save-temps -- 2>&1 | FileCheck %s
+# CHECK-NOT: error: no expected directives found: consider use of 'expected-no-diagnostics'
+# RUN: not ls %t.ii