[AST] Remove implicit includes from introspection generation.

[njames93]

Summary

Run ast dump twice with implicit includes defined and not defined. This increases coverage as some systems require them, others will fail with them.

[njames93]

Factor out changes

[njames93]

The old impl for me was resulting in these kinds of errors during parsing.

/usr/include/c++/10/cstdint:52:11: error: no member named 'int_fast8_t' in the global namespace
  using ::int_fast8_t;
        ~~^

After removing the implicit includes, these errors went away and generation completed as intended.

[njames93]

Run 2 invocations, one with implicit includes, one without them. If either has no errors the json will be generated, otherwise an empty implementation will be emitted.
Some systems seem to require the implicit includes, others, including mine, will fail with them defined.

[steveire]

In D100319#2697017, @njames93 wrote:

Run 2 invocations, one with implicit includes, one without them. If either has no errors the json will be generated, otherwise an empty implementation will be emitted.
Some systems seem to require the implicit includes, others, including mine, will fail with them defined.

Instead of running it twice, it would be better if we understood better what the problem is.

I can't reproduce any problem with the implicit includes present. Can you reduce the code being compiled by the tool to find out why their presence causes a problem?

clang/lib/Tooling/CMakeLists.txt

else()
   # file is generated by clang-ast-dump and generate_cxx_src_locs.py
   # generates the same API, but with a trivial implementation.
   option(CLANG_TOOLING_BUILD_AST_INTROSPECTION "Enable AST introspection" TRUE)
 
   set(skip_expensive_processing $<OR:$<CONFIG:Debug>,$<NOT:$<BOOL:${CLANG_TOOLING_BUILD_AST_INTROSPECTION}>>>)
 
   set(implicitDirs)
   foreach(implicitDir ${CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES})
-    list(APPEND implicitDirs -I ${implicitDir})
+    list(APPEND implicitDirs -implicit-include ${implicitDir})
   endforeach()
 
   add_custom_command(
       COMMENT Generate ASTNodeAPI.json
       OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTNodeAPI.json
       DEPENDS clang-ast-dump clang-resource-headers
       COMMAND
       $<TARGET_FILE:clang-ast-dump>

clang/lib/Tooling/DumpTool/ClangSrcLocDump.cpp

 using namespace clang::tooling;
 using namespace clang;
 using namespace llvm;
 
 static cl::list<std::string> IncludeDirectories(
     "I", cl::desc("Include directories to use while compiling"),
     cl::value_desc("directory"), cl::Required, cl::OneOrMore, cl::Prefix);
 
+static cl::list<std::string> ImplicitIncludeDirectories(
+    "implicit-include", cl::desc("Include directories to use while compiling"),
+    cl::value_desc("directory"), cl::OneOrMore, cl::Prefix);
+
 static cl::opt<bool>
     SkipProcessing("skip-processing",
                    cl::desc("Avoid processing the AST header file"),
                    cl::Required, cl::value_desc("bool"));
 
 static cl::opt<std::string> JsonOutputPath("json-output-path",
                                            cl::desc("json output path"),
                                            cl::Required,
                                            cl::value_desc("path"));
 
 class ASTSrcLocGenerationAction : public clang::ASTFrontendAction {
 public:
   ASTSrcLocGenerationAction() : Processor(JsonOutputPath) {}
 
   void ExecuteAction() override {
     clang::ASTFrontendAction::ExecuteAction();
-    if (getCompilerInstance().getDiagnostics().getNumErrors() > 0)
-      Processor.generateEmpty();
-    else
+    if (getCompilerInstance().getDiagnostics().getNumErrors() == 0) {
       Processor.generate();
+      HasWritten = true;
+    }
   }
 
   std::unique_ptr<clang::ASTConsumer>
   CreateASTConsumer(clang::CompilerInstance &Compiler,
                     llvm::StringRef File) override {
     return Processor.createASTConsumer(Compiler, File);
   }
 
+  bool hasWritten() const { return HasWritten; }
+
 private:
   ASTSrcLocProcessor Processor;
+  bool HasWritten = false;
 };
 
 static const char Filename[] = "ASTTU.cpp";
 
 int main(int argc, const char **argv) {
 
   cl::ParseCommandLineOptions(argc, argv);
 
   if (SkipProcessing) {
     std::error_code EC;
     llvm::raw_fd_ostream JsonOut(JsonOutputPath, EC, llvm::sys::fs::F_Text);
     if (EC)
       return 1;
     JsonOut << formatv("{0:2}", llvm::json::Value(llvm::json::Object()));
     return 0;
   }
 
-  std::vector<std::string> Args;
-  Args.push_back("-cc1");
-
-  llvm::transform(IncludeDirectories, std::back_inserter(Args),
-                  [](const std::string &IncDir) { return "-I" + IncDir; });
-
-  Args.push_back("-fsyntax-only");
-  Args.push_back(Filename);
-
-  std::vector<const char *> Argv(Args.size(), nullptr);
-  llvm::transform(Args, Argv.begin(),
-                  [](const std::string &Arg) { return Arg.c_str(); });
+  llvm::BumpPtrAllocator Alloc;
+  unsigned long Argc =
+      IncludeDirectories.size() + ImplicitIncludeDirectories.size() + 3;
+  MutableArrayRef<const char *> Argv(Alloc.Allocate<const char *>(Argc), Argc);
+  Argv[0] = "-cc1";
+  Argv[1] = "-fsyntax-only";
+  Argv[2] = Filename;
+  const char **Iter = Argv.drop_front(3).begin();
+  auto AddArg = [&Iter, &Alloc](StringRef Prefix, StringRef Item) {
+    char *Ptr = Alloc.Allocate<char>(Prefix.size() + Item.size() + 1);
+    std::memcpy(Ptr, Prefix.data(), Prefix.size());
+    std::memcpy(Ptr + Prefix.size(), Item.data(), Item.size());
+    Ptr[Prefix.size() + Item.size()] = '\0';
+    *Iter++ = Ptr;
+  };
+  for (auto &IncludeDir : IncludeDirectories)
+    AddArg("-I", IncludeDir);
+  for (auto &IncludeDir : ImplicitIncludeDirectories)
+    AddArg("-isystem", IncludeDir);
 
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
   unsigned MissingArgIndex, MissingArgCount;
   auto Opts = driver::getDriverOptTable();
-  auto ParsedArgs = Opts.ParseArgs(llvm::makeArrayRef(Argv).slice(1),
-                                   MissingArgIndex, MissingArgCount);
+  auto ParsedArgs =
+      Opts.ParseArgs(Argv.drop_front(), MissingArgIndex, MissingArgCount);
   ParseDiagnosticArgs(*DiagOpts, ParsedArgs);
 
   // Don't output diagnostics, because common scenarios such as
   // cross-compiling fail with diagnostics.  This is not fatal, but
   // just causes attempts to use the introspection API to return no data.
   TextDiagnosticPrinter DiagnosticPrinter(llvm::nulls(), &*DiagOpts);
   DiagnosticsEngine Diagnostics(
       IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
       &DiagnosticPrinter, false);
 
   auto *OFS = new llvm::vfs::OverlayFileSystem(vfs::getRealFileSystem());
 
   auto *MemFS = new llvm::vfs::InMemoryFileSystem();
   OFS->pushOverlay(MemFS);
   MemFS->addFile(Filename, 0,
                  MemoryBuffer::getMemBuffer("#include \"clang/AST/AST.h\"\n"));
 
   auto Files = llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions(), OFS);
 
   auto Driver = std::make_unique<driver::Driver>(
       "clang", llvm::sys::getDefaultTargetTriple(), Diagnostics,
       "ast-api-dump-tool", OFS);
 
+  SmallVector<ArrayRef<const char *>, 2> Commands;
+  Commands.push_back(Argv);
+  if (!ImplicitIncludeDirectories.empty())
+    Commands.push_back(Argv.drop_back(ImplicitIncludeDirectories.size()));
+  bool BadCommand = true;
+
+  for (auto &Command : Commands) {
     std::unique_ptr<clang::driver::Compilation> Comp(
-      Driver->BuildCompilation(llvm::makeArrayRef(Argv)));
+        Driver->BuildCompilation(Command));
     if (!Comp)
-    return 1;
-
+      continue;
     const auto &Jobs = Comp->getJobs();
-  if (Jobs.size() != 1 || !isa<driver::Command>(*Jobs.begin())) {
-    SmallString<256> error_msg;
-    llvm::raw_svector_ostream error_stream(error_msg);
-    Jobs.Print(error_stream, "; ", true);
-    return 1;
-  }
+    if (Jobs.size() != 1 || !isa<driver::Command>(*Jobs.begin()))
+      continue;
 
+    BadCommand = false;
     const auto &Cmd = cast<driver::Command>(*Jobs.begin());
     const llvm::opt::ArgStringList &CC1Args = Cmd.getArguments();
-
-  auto Invocation = std::make_unique<CompilerInvocation>();
-  CompilerInvocation::CreateFromArgs(*Invocation, CC1Args, Diagnostics);
-
-  CompilerInstance Compiler(std::make_shared<clang::PCHContainerOperations>());
-  Compiler.setInvocation(std::move(Invocation));
+    CompilerInstance Compiler(
+        std::make_shared<clang::PCHContainerOperations>());
 
     Compiler.createDiagnostics(&DiagnosticPrinter, false);
     if (!Compiler.hasDiagnostics())
       return 1;
 
+    Compiler.setFileManager(Files.get());
+    Compiler.createSourceManager(*Files);
+
+    auto Invocation = std::make_unique<CompilerInvocation>();
+    CompilerInvocation::CreateFromArgs(*Invocation, CC1Args, Diagnostics);
+    Compiler.setInvocation(std::move(Invocation));
     // Suppress "2 errors generated" or similar messages
     Compiler.getDiagnosticOpts().ShowCarets = false;
-  Compiler.createSourceManager(*Files);
-  Compiler.setFileManager(Files.get());
 
     ASTSrcLocGenerationAction ScopedToolAction;
     Compiler.ExecuteAction(ScopedToolAction);
 
-  Files->clearStatCache();
-
+    if (ScopedToolAction.hasWritten())
       return 0;
+    Compiler.getDiagnostics().Reset();
+    Compiler.getDiagnostics().getClient()->clear();
+  }
+  ASTSrcLocProcessor(JsonOutputPath).generateEmpty();
+  return BadCommand;
 }