Add a tool executor that runs actions on all TUs in the compilation database.

[ioeric]

Summary

Tool results are deduplicated by the result key.

[hokein]

LGTM.

include/clang/Tooling/AllTUsExecution.h

35

consider using a default parameter (= 0)? use the llvm::hardware_concurrency by default, and you don't have to pass the 0 in the unittest.

lib/Tooling/AllTUsExecution.cpp

158

also document all results are stored in memory, the result size should be suitable to be loaded in memory.

unittests/Tooling/ExecutionTest.cpp

264

nit: ++i.

[ioeric]

Address review comments.

include/clang/Tooling/AllTUsExecution.h

35

The constructors aren't usually used directly (outside of unit tests; interface design shouldn't be driven by tests), so I didn't use default parameters, since they are usually disliked...

unittests/Tooling/ExecutionTest.cpp

264

Done. (I don't think this matters for simple types tho...)

include/clang/Tooling/AllTUsExecution.h

+//===--- AllTUsExecution.h - Execute actions on all TUs. -*- C++ --------*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines a tool executor that runs given actions on all TUs in the
+//  compilation database. Tool results are deuplicated by the result key.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H
+#define LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H
+
+#include "clang/Tooling/ArgumentsAdjusters.h"
+#include "clang/Tooling/Execution.h"
+
+namespace clang {
+namespace tooling {
+
+/// \brief Executes given frontend actions on all files/TUs in the compilation
+/// database. The final results will be deduplicated by the result key.
+class AllTUsToolExecutor : public ToolExecutor {
+public:
+  static const char *ExecutorName;
+
+  /// \brief Init with \p CompilationDatabase.
+  /// This uses \p ThreadCount threads to exececute the actions on all files in
+  /// parallel. If \p ThreadCount is 0, this uses `llvm::hardware_concurrency`.
+  AllTUsToolExecutor(const CompilationDatabase &Compilations,
+                     unsigned ThreadCount,
+                     std::shared_ptr<PCHContainerOperations> PCHContainerOps =
+                         std::make_shared<PCHContainerOperations>());
+
+  /// \brief Init with \p CommonOptionsParser. This is expected to be used by
+  /// `createExecutorFromCommandLineArgs` based on commandline options.
+  ///
+  /// The executor takes ownership of \p Options.
+  AllTUsToolExecutor(CommonOptionsParser Options, unsigned ThreadCount,
+                     std::shared_ptr<PCHContainerOperations> PCHContainerOps =
+                         std::make_shared<PCHContainerOperations>());
+
+  StringRef getExecutorName() const override { return ExecutorName; }
+
+  using ToolExecutor::execute;
+
+  llvm::Error
+  execute(llvm::ArrayRef<
+          std::pair<std::unique_ptr<FrontendActionFactory>, ArgumentsAdjuster>>
+              Actions) override;
+
+  ExecutionContext *getExecutionContext() override { return &Context; };
+
+  ToolResults *getToolResults() override { return Results.get(); }
+
+  void mapVirtualFile(StringRef FilePath, StringRef Content) override {
+    OverlayFiles[FilePath] = Content;
+  }
+
+private:
+  // Used to store the parser when the executor is initialized with parser.
+  llvm::Optional<CommonOptionsParser> OptionsParser;
+  const CompilationDatabase &Compilations;
+  std::unique_ptr<ToolResults> Results;
+  ExecutionContext Context;
+  llvm::StringMap<std::string> OverlayFiles;
+  unsigned ThreadCount;
+};
+
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H

lib/Tooling/AllTUsExecution.cpp

+//===- lib/Tooling/AllTUsExecution.cpp - Execute actions on all TUs. ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/AllTUsExecution.h"
+#include "clang/Tooling/ToolExecutorPluginRegistry.h"
+#include "llvm/Support/ThreadPool.h"
+
+namespace clang {
+namespace tooling {
+
+const char *AllTUsToolExecutor::ExecutorName = "AllTUsToolExecutor";
+
+namespace {
+llvm::Error make_string_error(const llvm::Twine &Message) {
+  return llvm::make_error<llvm::StringError>(Message,
+                                             llvm::inconvertibleErrorCode());
+}
+
+ArgumentsAdjuster getDefaultArgumentsAdjusters() {
+  return combineAdjusters(
+      getClangStripOutputAdjuster(),
+      combineAdjusters(getClangSyntaxOnlyAdjuster(),
+                       getClangStripDependencyFileAdjuster()));
+}
+
+class ThreadSafeToolResults : public ToolResults {
+public:
+  void addResult(StringRef Key, StringRef Value) override {
+    std::unique_lock<std::mutex> LockGuard(Mutex);
+    Results[Key] = Value;
+  }
+
+  std::vector<std::pair<std::string, std::string>> AllKVResults() override {
+    std::vector<std::pair<std::string, std::string>> KVs;
+    for (const auto &Pair : Results)
+      KVs.emplace_back(Pair.first().str(), Pair.second);
+    return KVs;
+  }
+
+  void forEachResult(llvm::function_ref<void(StringRef Key, StringRef Value)>
+                         Callback) override {
+    for (const auto &Pair : Results)
+      Callback(Pair.first(), Pair.second);
+  }
+
+private:
+  llvm::StringMap<std::string> Results;
+  std::mutex Mutex;
+};
+
+} // namespace
+
+AllTUsToolExecutor::AllTUsToolExecutor(
+    const CompilationDatabase &Compilations, unsigned ThreadCount,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps)
+    : Compilations(Compilations), Results(new ThreadSafeToolResults),
+      Context(Results.get()), ThreadCount(ThreadCount) {}
+
+AllTUsToolExecutor::AllTUsToolExecutor(
+    CommonOptionsParser Options, unsigned ThreadCount,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps)
+    : OptionsParser(std::move(Options)),
+      Compilations(OptionsParser->getCompilations()),
+      Results(new ThreadSafeToolResults), Context(Results.get()),
+      ThreadCount(ThreadCount) {}
+
+llvm::Error AllTUsToolExecutor::execute(
+    llvm::ArrayRef<
+        std::pair<std::unique_ptr<FrontendActionFactory>, ArgumentsAdjuster>>
+        Actions) {
+  if (Actions.empty())
+    return make_string_error("No action to execute.");
+
+  if (Actions.size() != 1)
+    return make_string_error(
+        "Only support executing exactly 1 action at this point.");
+
+  std::string ErrorMsg;
+  std::mutex TUMutex;
+  auto AppendError = [&](llvm::Twine Err) {
+    std::unique_lock<std::mutex> LockGuard(TUMutex);
+    ErrorMsg += Err.str();
+  };
+
+  auto Log = [&](llvm::Twine Msg) {
+    std::unique_lock<std::mutex> LockGuard(TUMutex);
+    llvm::errs() << Msg.str() << "\n";
+  };
+
+  auto Files = Compilations.getAllFiles();
+  // Add a counter to track the progress.
+  const std::string TotalNumStr = std::to_string(Files.size());
+  unsigned Counter = 0;
+  auto Count = [&]() {
+    std::unique_lock<std::mutex> LockGuard(TUMutex);
+    return ++Counter;
+  };
+
+  auto &Action = Actions.front();
+
+  {
+    llvm::ThreadPool Pool(ThreadCount == 0 ? llvm::hardware_concurrency()
+                                           : ThreadCount);
+
+    for (std::string File : Files) {
+      Pool.async(
+          [&](std::string Path) {
+            Log("[" + std::to_string(Count()) + "/" + TotalNumStr +
+                "] Processing file " + Path);
+            ClangTool Tool(Compilations, {Path});
+            Tool.appendArgumentsAdjuster(Action.second);
+            Tool.appendArgumentsAdjuster(getDefaultArgumentsAdjusters());
+            for (const auto &FileAndContent : OverlayFiles)
+              Tool.mapVirtualFile(FileAndContent.first(),
+                                  FileAndContent.second);
+            if (Tool.run(Action.first.get()))
+              AppendError(llvm::Twine("Failed to run action on ") + Path +
+                          "\n");
+          },
+          File);
+    }
+  }
+
+  if (!ErrorMsg.empty())
+    return make_string_error(ErrorMsg);
+
+  return llvm::Error::success();
+}
+
+static llvm::cl::opt<unsigned> ExecutorConcurrency(
+    "execute-concurrency",
+    llvm::cl::desc("The number of threads used to process all files in "
+                   "parallel. Set to 0 for hardware concurrency."),
+    llvm::cl::init(0));
+
+class AllTUsToolExecutorPlugin : public ToolExecutorPlugin {
+public:
+  llvm::Expected<std::unique_ptr<ToolExecutor>>
+  create(CommonOptionsParser &OptionsParser) override {
+    if (OptionsParser.getSourcePathList().empty())
+      return make_string_error(
+          "[AllTUsToolExecutorPlugin] Please provide a directory/file path in "
+          "the compilation database.");
+    return llvm::make_unique<AllTUsToolExecutor>(std::move(OptionsParser),
+                                                 ExecutorConcurrency);
+  }
+};
+
+static ToolExecutorPluginRegistry::Add<AllTUsToolExecutorPlugin>
+    X("all-TUs",
+      "Runs FrontendActions on all TUs in the compilation database. "
+      "Tool results are deduplicated by the result key and stored in memory.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the plugin.
+volatile int AllTUsToolExecutorAnchorSource = 0;
+
+} // end namespace tooling
+} // end namespace clang

lib/Tooling/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Option
   Support
   )
 
 add_subdirectory(Core)
 add_subdirectory(Refactoring)
 add_subdirectory(ASTDiff)
 
 add_clang_library(clangTooling
+  AllTUsExecution.cpp
   ArgumentsAdjusters.cpp
   CommonOptionsParser.cpp
   CompilationDatabase.cpp
   Execution.cpp
   FileMatchTrie.cpp
   FixIt.cpp
   JSONCompilationDatabase.cpp
   Refactoring.cpp

lib/Tooling/Execution.cpp

 createExecutorFromCommandLineArgs(int &argc, const char **argv,
                                   llvm::cl::OptionCategory &Category,
                                   const char *Overview) {
   return internal::createExecutorFromCommandLineArgsImpl(argc, argv, Category,
                                                          Overview);
 }
 
 // This anchor is used to force the linker to link in the generated object file
-// and thus register the StandaloneToolExecutorPlugin.
+// and thus register the StandaloneToolExecutorPlugin etc.
 extern volatile int StandaloneToolExecutorAnchorSource;
+extern volatile int AllTUsToolExecutorAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED StandaloneToolExecutorAnchorDest =
     StandaloneToolExecutorAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED AllTUsToolExecutorAnchorDest =
+    AllTUsToolExecutorAnchorSource;
 
 } // end namespace tooling
 } // end namespace clang

unittests/Tooling/ExecutionTest.cpp

 //===- unittest/Tooling/ExecutionTest.cpp - Tool execution tests. --------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Tooling/Execution.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/AllTUsExecution.h"
 #include "clang/Tooling/CompilationDatabase.h"
-#include "clang/Tooling/Execution.h"
 #include "clang/Tooling/StandaloneExecution.h"
 #include "clang/Tooling/ToolExecutorPluginRegistry.h"
 #include "clang/Tooling/Tooling.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <algorithm>
 #include <string>
 
 namespace clang {
 namespace tooling {
 
 namespace {
TEST(StandaloneToolTest, SimpleActionWithResult) {
   ASSERT_EQ(KVs.size(), 1u);
   EXPECT_EQ("f", KVs[0].first);
   EXPECT_EQ("1", KVs[0].second);
 
   Executor.getToolResults()->forEachResult(
       [](StringRef, StringRef Value) { EXPECT_EQ("1", Value); });
 }
 
+class FixedCompilationDatabaseWithFiles : public CompilationDatabase {
+public:
+  FixedCompilationDatabaseWithFiles(Twine Directory,
+                                    ArrayRef<std::string> Files,
+                                    ArrayRef<std::string> CommandLine)
+      : FixedCompilations(Directory, CommandLine), Files(Files) {}
+
+  std::vector<CompileCommand>
+  getCompileCommands(StringRef FilePath) const override {
+    return FixedCompilations.getCompileCommands(FilePath);
+  }
+
+  std::vector<std::string> getAllFiles() const override { return Files; }
+
+private:
+  FixedCompilationDatabase FixedCompilations;
+  std::vector<std::string> Files;
+};
+
+MATCHER_P(Named, Name, "") { return arg.first == Name; }
+
+TEST(AllTUsToolTest, AFewFiles) {
+  FixedCompilationDatabaseWithFiles Compilations(".", {"a.cc", "b.cc", "c.cc"},
+                                                 std::vector<std::string>());
+  AllTUsToolExecutor Executor(Compilations, /*ThreadCount=*/0);
+  Executor.mapVirtualFile("a.cc", "void x() {}");
+  Executor.mapVirtualFile("b.cc", "void y() {}");
+  Executor.mapVirtualFile("c.cc", "void z() {}");
+
+  auto Err = Executor.execute(std::unique_ptr<FrontendActionFactory>(
+      new ReportResultActionFactory(Executor.getExecutionContext())));
+  ASSERT_TRUE(!Err);
+  EXPECT_THAT(
+      Executor.getToolResults()->AllKVResults(),
+      ::testing::UnorderedElementsAre(Named("x"), Named("y"), Named("z")));
+}
+
+TEST(AllTUsToolTest, ManyFiles) {
+  unsigned NumFiles = 100;
+  std::vector<std::string> Files;
+  std::map<std::string, std::string> FileToContent;
+  std::vector<std::string> ExpectedSymbols;
+  for (unsigned i = 1; i <= NumFiles; ++i) {
+    std::string File = "f" + std::to_string(i) + ".cc";
+    std::string Symbol = "looong_function_name_" + std::to_string(i);
+    Files.push_back(File);
+    FileToContent[File] = "void " + Symbol + "() {}";
+    ExpectedSymbols.push_back(Symbol);
+  }
+  FixedCompilationDatabaseWithFiles Compilations(".", Files,
+                                                 std::vector<std::string>());
+  AllTUsToolExecutor Executor(Compilations, /*ThreadCount=*/0);
+  for (const auto &FileAndContent : FileToContent) {
+    Executor.mapVirtualFile(FileAndContent.first, FileAndContent.second);
+  }
+
+  auto Err = Executor.execute(std::unique_ptr<FrontendActionFactory>(
+      new ReportResultActionFactory(Executor.getExecutionContext())));
+  ASSERT_TRUE(!Err);
+  std::vector<std::string> Results;
+  Executor.getToolResults()->forEachResult(
+      [&](StringRef Name, StringRef) { Results.push_back(Name); });
+  EXPECT_THAT(ExpectedSymbols, ::testing::UnorderedElementsAreArray(Results));
+}
+
 } // end namespace tooling
 } // end namespace clang