Index: include/clang/Tooling/AllTUsExecution.h =================================================================== --- include/clang/Tooling/AllTUsExecution.h +++ include/clang/Tooling/AllTUsExecution.h @@ -0,0 +1,76 @@ +//===--- AllTUsExecution.h - Execute actions on all TUs. -*- C++ --------*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines a tool executor that runs given actions on all TUs in the +// compilation database. Tool results are deuplicated by the result key. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H +#define LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H + +#include "clang/Tooling/ArgumentsAdjusters.h" +#include "clang/Tooling/Execution.h" + +namespace clang { +namespace tooling { + +/// \brief Executes given frontend actions on all files/TUs in the compilation +/// database. The final results will be deduplicated by the result key. +class AllTUsToolExecutor : public ToolExecutor { +public: + static const char *ExecutorName; + + /// \brief Init with \p CompilationDatabase. + /// This uses \p ThreadCount threads to exececute the actions on all files in + /// parallel. If \p ThreadCount is 0, this uses `llvm::hardware_concurrency`. + AllTUsToolExecutor(const CompilationDatabase &Compilations, + unsigned ThreadCount, + std::shared_ptr PCHContainerOps = + std::make_shared()); + + /// \brief Init with \p CommonOptionsParser. This is expected to be used by + /// `createExecutorFromCommandLineArgs` based on commandline options. + /// + /// The executor takes ownership of \p Options. + AllTUsToolExecutor(CommonOptionsParser Options, unsigned ThreadCount, + std::shared_ptr PCHContainerOps = + std::make_shared()); + + StringRef getExecutorName() const override { return ExecutorName; } + + using ToolExecutor::execute; + + llvm::Error + execute(llvm::ArrayRef< + std::pair, ArgumentsAdjuster>> + Actions) override; + + ExecutionContext *getExecutionContext() override { return &Context; }; + + ToolResults *getToolResults() override { return Results.get(); } + + void mapVirtualFile(StringRef FilePath, StringRef Content) override { + OverlayFiles[FilePath] = Content; + } + +private: + // Used to store the parser when the executor is initialized with parser. + llvm::Optional OptionsParser; + const CompilationDatabase &Compilations; + std::unique_ptr Results; + ExecutionContext Context; + llvm::StringMap OverlayFiles; + unsigned ThreadCount; +}; + +} // end namespace tooling +} // end namespace clang + +#endif // LLVM_CLANG_TOOLING_ALLTUSEXECUTION_H Index: lib/Tooling/AllTUsExecution.cpp =================================================================== --- lib/Tooling/AllTUsExecution.cpp +++ lib/Tooling/AllTUsExecution.cpp @@ -0,0 +1,165 @@ +//===- lib/Tooling/AllTUsExecution.cpp - Execute actions on all TUs. ------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/AllTUsExecution.h" +#include "clang/Tooling/ToolExecutorPluginRegistry.h" +#include "llvm/Support/ThreadPool.h" + +namespace clang { +namespace tooling { + +const char *AllTUsToolExecutor::ExecutorName = "AllTUsToolExecutor"; + +namespace { +llvm::Error make_string_error(const llvm::Twine &Message) { + return llvm::make_error(Message, + llvm::inconvertibleErrorCode()); +} + +ArgumentsAdjuster getDefaultArgumentsAdjusters() { + return combineAdjusters( + getClangStripOutputAdjuster(), + combineAdjusters(getClangSyntaxOnlyAdjuster(), + getClangStripDependencyFileAdjuster())); +} + +class ThreadSafeToolResults : public ToolResults { +public: + void addResult(StringRef Key, StringRef Value) override { + std::unique_lock LockGuard(Mutex); + Results[Key] = Value; + } + + std::vector> AllKVResults() override { + std::vector> KVs; + for (const auto &Pair : Results) + KVs.emplace_back(Pair.first().str(), Pair.second); + return KVs; + } + + void forEachResult(llvm::function_ref + Callback) override { + for (const auto &Pair : Results) + Callback(Pair.first(), Pair.second); + } + +private: + llvm::StringMap Results; + std::mutex Mutex; +}; + +} // namespace + +AllTUsToolExecutor::AllTUsToolExecutor( + const CompilationDatabase &Compilations, unsigned ThreadCount, + std::shared_ptr PCHContainerOps) + : Compilations(Compilations), Results(new ThreadSafeToolResults), + Context(Results.get()), ThreadCount(ThreadCount) {} + +AllTUsToolExecutor::AllTUsToolExecutor( + CommonOptionsParser Options, unsigned ThreadCount, + std::shared_ptr PCHContainerOps) + : OptionsParser(std::move(Options)), + Compilations(OptionsParser->getCompilations()), + Results(new ThreadSafeToolResults), Context(Results.get()), + ThreadCount(ThreadCount) {} + +llvm::Error AllTUsToolExecutor::execute( + llvm::ArrayRef< + std::pair, ArgumentsAdjuster>> + Actions) { + if (Actions.empty()) + return make_string_error("No action to execute."); + + if (Actions.size() != 1) + return make_string_error( + "Only support executing exactly 1 action at this point."); + + std::string ErrorMsg; + std::mutex TUMutex; + auto AppendError = [&](llvm::Twine Err) { + std::unique_lock LockGuard(TUMutex); + ErrorMsg += Err.str(); + }; + + auto Log = [&](llvm::Twine Msg) { + std::unique_lock LockGuard(TUMutex); + llvm::errs() << Msg.str() << "\n"; + }; + + auto Files = Compilations.getAllFiles(); + // Add a counter to track the progress. + const std::string TotalNumStr = std::to_string(Files.size()); + unsigned Counter = 0; + auto Count = [&]() { + std::unique_lock LockGuard(TUMutex); + return ++Counter; + }; + + auto &Action = Actions.front(); + + { + llvm::ThreadPool Pool(ThreadCount == 0 ? llvm::hardware_concurrency() + : ThreadCount); + + for (std::string File : Files) { + Pool.async( + [&](std::string Path) { + Log("[" + std::to_string(Count()) + "/" + TotalNumStr + + "] Processing file " + Path); + ClangTool Tool(Compilations, {Path}); + Tool.appendArgumentsAdjuster(Action.second); + Tool.appendArgumentsAdjuster(getDefaultArgumentsAdjusters()); + for (const auto &FileAndContent : OverlayFiles) + Tool.mapVirtualFile(FileAndContent.first(), + FileAndContent.second); + if (Tool.run(Action.first.get())) + AppendError(llvm::Twine("Failed to run action on ") + Path + + "\n"); + }, + File); + } + } + + if (!ErrorMsg.empty()) + return make_string_error(ErrorMsg); + + return llvm::Error::success(); +} + +static llvm::cl::opt ExecutorConcurrency( + "execute-concurrency", + llvm::cl::desc("The number of threads used to process all files in " + "parallel. Set to 0 for hardware concurrency."), + llvm::cl::init(0)); + +class AllTUsToolExecutorPlugin : public ToolExecutorPlugin { +public: + llvm::Expected> + create(CommonOptionsParser &OptionsParser) override { + if (OptionsParser.getSourcePathList().empty()) + return make_string_error( + "[AllTUsToolExecutorPlugin] Please provide a directory/file path in " + "the compilation database."); + return llvm::make_unique(std::move(OptionsParser), + ExecutorConcurrency); + } +}; + +static ToolExecutorPluginRegistry::Add + X("all-TUs", + "Runs FrontendActions on all TUs in the compilation database. " + "Tool results are deduplicated by the result key and stored in memory."); + +// This anchor is used to force the linker to link in the generated object file +// and thus register the plugin. +volatile int AllTUsToolExecutorAnchorSource = 0; + +} // end namespace tooling +} // end namespace clang Index: lib/Tooling/CMakeLists.txt =================================================================== --- lib/Tooling/CMakeLists.txt +++ lib/Tooling/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(ASTDiff) add_clang_library(clangTooling + AllTUsExecution.cpp ArgumentsAdjusters.cpp CommonOptionsParser.cpp CompilationDatabase.cpp Index: lib/Tooling/Execution.cpp =================================================================== --- lib/Tooling/Execution.cpp +++ lib/Tooling/Execution.cpp @@ -96,10 +96,13 @@ } // This anchor is used to force the linker to link in the generated object file -// and thus register the StandaloneToolExecutorPlugin. +// and thus register the StandaloneToolExecutorPlugin etc. extern volatile int StandaloneToolExecutorAnchorSource; +extern volatile int AllTUsToolExecutorAnchorSource; static int LLVM_ATTRIBUTE_UNUSED StandaloneToolExecutorAnchorDest = StandaloneToolExecutorAnchorSource; +static int LLVM_ATTRIBUTE_UNUSED AllTUsToolExecutorAnchorDest = + AllTUsToolExecutorAnchorSource; } // end namespace tooling } // end namespace clang Index: unittests/Tooling/ExecutionTest.cpp =================================================================== --- unittests/Tooling/ExecutionTest.cpp +++ unittests/Tooling/ExecutionTest.cpp @@ -7,17 +7,19 @@ // //===----------------------------------------------------------------------===// +#include "clang/Tooling/Execution.h" #include "clang/AST/ASTConsumer.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/Frontend/ASTUnit.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/AllTUsExecution.h" #include "clang/Tooling/CompilationDatabase.h" -#include "clang/Tooling/Execution.h" #include "clang/Tooling/StandaloneExecution.h" #include "clang/Tooling/ToolExecutorPluginRegistry.h" #include "clang/Tooling/Tooling.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include #include @@ -217,5 +219,70 @@ [](StringRef, StringRef Value) { EXPECT_EQ("1", Value); }); } +class FixedCompilationDatabaseWithFiles : public CompilationDatabase { +public: + FixedCompilationDatabaseWithFiles(Twine Directory, + ArrayRef Files, + ArrayRef CommandLine) + : FixedCompilations(Directory, CommandLine), Files(Files) {} + + std::vector + getCompileCommands(StringRef FilePath) const override { + return FixedCompilations.getCompileCommands(FilePath); + } + + std::vector getAllFiles() const override { return Files; } + +private: + FixedCompilationDatabase FixedCompilations; + std::vector Files; +}; + +MATCHER_P(Named, Name, "") { return arg.first == Name; } + +TEST(AllTUsToolTest, AFewFiles) { + FixedCompilationDatabaseWithFiles Compilations(".", {"a.cc", "b.cc", "c.cc"}, + std::vector()); + AllTUsToolExecutor Executor(Compilations, /*ThreadCount=*/0); + Executor.mapVirtualFile("a.cc", "void x() {}"); + Executor.mapVirtualFile("b.cc", "void y() {}"); + Executor.mapVirtualFile("c.cc", "void z() {}"); + + auto Err = Executor.execute(std::unique_ptr( + new ReportResultActionFactory(Executor.getExecutionContext()))); + ASSERT_TRUE(!Err); + EXPECT_THAT( + Executor.getToolResults()->AllKVResults(), + ::testing::UnorderedElementsAre(Named("x"), Named("y"), Named("z"))); +} + +TEST(AllTUsToolTest, ManyFiles) { + unsigned NumFiles = 100; + std::vector Files; + std::map FileToContent; + std::vector ExpectedSymbols; + for (unsigned i = 1; i <= NumFiles; ++i) { + std::string File = "f" + std::to_string(i) + ".cc"; + std::string Symbol = "looong_function_name_" + std::to_string(i); + Files.push_back(File); + FileToContent[File] = "void " + Symbol + "() {}"; + ExpectedSymbols.push_back(Symbol); + } + FixedCompilationDatabaseWithFiles Compilations(".", Files, + std::vector()); + AllTUsToolExecutor Executor(Compilations, /*ThreadCount=*/0); + for (const auto &FileAndContent : FileToContent) { + Executor.mapVirtualFile(FileAndContent.first, FileAndContent.second); + } + + auto Err = Executor.execute(std::unique_ptr( + new ReportResultActionFactory(Executor.getExecutionContext()))); + ASSERT_TRUE(!Err); + std::vector Results; + Executor.getToolResults()->forEachResult( + [&](StringRef Name, StringRef) { Results.push_back(Name); }); + EXPECT_THAT(ExpectedSymbols, ::testing::UnorderedElementsAreArray(Results)); +} + } // end namespace tooling } // end namespace clang