Index: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h =================================================================== --- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -0,0 +1,58 @@ +//===- DependencyScanningWorker.h - clang-scan-deps worker ===---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H +#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H + +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Basic/LLVM.h" +#include "clang/Frontend/PCHContainerOperations.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include + +namespace clang { +namespace tooling { +namespace dependencies { + +/// An individual dependency scanning worker that is able to run on its own +/// thread. +/// +/// The worker computes the dependencies for the input files by preprocessing +/// sources either using a fast mode where the source files are minimized, or +/// using the regular processing run. +class DependencyScanningWorker { +public: + DependencyScanningWorker(); + + /// Print out the dependency information into a string using the dependency + /// file format that is specified in the options (-MD is the default) and + /// return it. + /// + /// \returns A \c StringError with the diagnostic output if clang errors + /// occurred, dependency file contents otherwise. + llvm::Expected getDependencyFile(const std::string &Input, + StringRef WorkingDirectory, + const CompilationDatabase &CDB); + +private: + IntrusiveRefCntPtr DiagOpts; + std::shared_ptr PCHContainerOps; + + /// The file system that is used by each worker when scanning for + /// dependencies. This filesystem persists accross multiple compiler + /// invocations. + llvm::IntrusiveRefCntPtr WorkerFS; +}; + +} // end namespace dependencies +} // end namespace tooling +} // end namespace clang + +#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H Index: cfe/trunk/include/clang/Tooling/Tooling.h =================================================================== --- cfe/trunk/include/clang/Tooling/Tooling.h +++ cfe/trunk/include/clang/Tooling/Tooling.h @@ -360,6 +360,10 @@ /// turn this off when running on multiple threads to avoid the raciness. void setRestoreWorkingDir(bool RestoreCWD); + /// Sets whether an error message should be printed out if an action fails. By + /// default, if an action fails, a message is printed out to stderr. + void setPrintErrorMessage(bool PrintErrorMessage); + /// Returns the file manager used in the tool. /// /// The file manager is shared between all translation units. @@ -386,6 +390,7 @@ DiagnosticConsumer *DiagConsumer = nullptr; bool RestoreCWD = true; + bool PrintErrorMessage = true; }; template Index: cfe/trunk/lib/Tooling/CMakeLists.txt =================================================================== --- cfe/trunk/lib/Tooling/CMakeLists.txt +++ cfe/trunk/lib/Tooling/CMakeLists.txt @@ -8,6 +8,7 @@ add_subdirectory(Refactoring) add_subdirectory(ASTDiff) add_subdirectory(Syntax) +add_subdirectory(DependencyScanning) add_clang_library(clangTooling AllTUsExecution.cpp Index: cfe/trunk/lib/Tooling/DependencyScanning/CMakeLists.txt =================================================================== --- cfe/trunk/lib/Tooling/DependencyScanning/CMakeLists.txt +++ cfe/trunk/lib/Tooling/DependencyScanning/CMakeLists.txt @@ -0,0 +1,22 @@ +set(LLVM_LINK_COMPONENTS + Core + Support + ) + +add_clang_library(clangDependencyScanning + DependencyScanningWorker.cpp + + DEPENDS + ClangDriverOptions + + LINK_LIBS + clangAST + clangBasic + clangDriver + clangFrontend + clangFrontendTool + clangLex + clangParse + clangSerialization + clangTooling +) Index: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp =================================================================== --- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -0,0 +1,149 @@ +//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "clang/Tooling/Tooling.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +namespace { + +/// Prints out all of the gathered dependencies into a string. +class DependencyPrinter : public DependencyFileGenerator { +public: + DependencyPrinter(std::unique_ptr Opts, + std::string &S) + : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), S(S) {} + + void finishedMainFile(DiagnosticsEngine &Diags) override { + llvm::raw_string_ostream OS(S); + outputDependencyFile(OS); + } + +private: + std::unique_ptr Opts; + std::string &S; +}; + +/// A proxy file system that doesn't call `chdir` when changing the working +/// directory of a clang tool. +class ProxyFileSystemWithoutChdir : public llvm::vfs::ProxyFileSystem { +public: + ProxyFileSystemWithoutChdir( + llvm::IntrusiveRefCntPtr FS) + : ProxyFileSystem(std::move(FS)) {} + + llvm::ErrorOr getCurrentWorkingDirectory() const override { + assert(!CWD.empty() && "empty CWD"); + return CWD; + } + + std::error_code setCurrentWorkingDirectory(const Twine &Path) override { + CWD = Path.str(); + return {}; + } + +private: + std::string CWD; +}; + +/// A clang tool that runs the preprocessor in a mode that's optimized for +/// dependency scanning for the given compiler invocation. +class DependencyScanningAction : public tooling::ToolAction { +public: + DependencyScanningAction(StringRef WorkingDirectory, + std::string &DependencyFileContents) + : WorkingDirectory(WorkingDirectory), + DependencyFileContents(DependencyFileContents) {} + + bool runInvocation(std::shared_ptr Invocation, + FileManager *FileMgr, + std::shared_ptr PCHContainerOps, + DiagnosticConsumer *DiagConsumer) override { + // Create a compiler instance to handle the actual work. + CompilerInstance Compiler(std::move(PCHContainerOps)); + Compiler.setInvocation(std::move(Invocation)); + FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; + Compiler.setFileManager(FileMgr); + + // Don't print 'X warnings and Y errors generated'. + Compiler.getDiagnosticOpts().ShowCarets = false; + // Create the compiler's actual diagnostics engine. + Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); + if (!Compiler.hasDiagnostics()) + return false; + + Compiler.createSourceManager(*FileMgr); + + // Create the dependency collector that will collect the produced + // dependencies. + // + // This also moves the existing dependency output options from the + // invocation to the collector. The options in the invocation are reset, + // which ensures that the compiler won't create new dependency collectors, + // and thus won't write out the extra '.d' files to disk. + auto Opts = llvm::make_unique( + std::move(Compiler.getInvocation().getDependencyOutputOpts())); + // We need at least one -MT equivalent for the generator to work. + if (Opts->Targets.empty()) + Opts->Targets = {"clang-scan-deps dependency"}; + Compiler.addDependencyCollector(std::make_shared( + std::move(Opts), DependencyFileContents)); + + auto Action = llvm::make_unique(); + const bool Result = Compiler.ExecuteAction(*Action); + FileMgr->clearStatCache(); + return Result; + } + +private: + StringRef WorkingDirectory; + /// The dependency file will be written to this string. + std::string &DependencyFileContents; +}; + +} // end anonymous namespace + +DependencyScanningWorker::DependencyScanningWorker() { + DiagOpts = new DiagnosticOptions(); + PCHContainerOps = std::make_shared(); + /// FIXME: Use the shared file system from the service for fast scanning + /// mode. + WorkerFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); +} + +llvm::Expected +DependencyScanningWorker::getDependencyFile(const std::string &Input, + StringRef WorkingDirectory, + const CompilationDatabase &CDB) { + // Capture the emitted diagnostics and report them to the client + // in the case of a failure. + std::string DiagnosticOutput; + llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); + TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.get()); + + WorkerFS->setCurrentWorkingDirectory(WorkingDirectory); + tooling::ClangTool Tool(CDB, Input, PCHContainerOps, WorkerFS); + Tool.clearArgumentsAdjusters(); + Tool.setRestoreWorkingDir(false); + Tool.setPrintErrorMessage(false); + Tool.setDiagnosticConsumer(&DiagPrinter); + std::string Output; + DependencyScanningAction Action(WorkingDirectory, Output); + if (Tool.run(&Action)) { + return llvm::make_error(DiagnosticsOS.str(), + llvm::inconvertibleErrorCode()); + } + return Output; +} Index: cfe/trunk/lib/Tooling/Tooling.cpp =================================================================== --- cfe/trunk/lib/Tooling/Tooling.cpp +++ cfe/trunk/lib/Tooling/Tooling.cpp @@ -517,7 +517,8 @@ if (!Invocation.run()) { // FIXME: Diagnostics should be used instead. - llvm::errs() << "Error while processing " << File << ".\n"; + if (PrintErrorMessage) + llvm::errs() << "Error while processing " << File << ".\n"; ProcessingFailed = true; } } @@ -569,6 +570,10 @@ this->RestoreCWD = RestoreCWD; } +void ClangTool::setPrintErrorMessage(bool PrintErrorMessage) { + this->PrintErrorMessage = PrintErrorMessage; +} + namespace clang { namespace tooling { Index: cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json =================================================================== --- cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json +++ cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json @@ -6,7 +6,7 @@ }, { "directory": "DIR", - "command": "clang -E -fsyntax-only DIR/regular_cdb.cpp -IInputs", + "command": "clang -E DIR/regular_cdb.cpp -IInputs", "file": "DIR/regular_cdb.cpp" } ] Index: cfe/trunk/test/ClangScanDeps/error.cpp =================================================================== --- cfe/trunk/test/ClangScanDeps/error.cpp +++ cfe/trunk/test/ClangScanDeps/error.cpp @@ -0,0 +1,21 @@ +// RUN: rm -rf %t.dir +// RUN: rm -rf %t.cdb +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/regular_cdb.cpp +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb +// +// RUN: not clang-scan-deps -compilation-database %t.cdb -j 1 2>%t.dir/errs +// RUN: echo EOF >> %t.dir/errs +// RUN: FileCheck %s --input-file %t.dir/errs + +#include "missing.h" + +// CHECK: Error while scanning dependencies +// CHECK-NEXT: error: no such file or directory: +// CHECK-NEXT: error: no input files +// CHECK-NEXT: error: +// CHECK-NEXT: Error while scanning dependencies +// CHECK-NEXT: fatal error: 'missing.h' file not found +// CHECK-NEXT: "missing.h" +// CHECK-NEXT: ^ +// CHECK-NEXT: EOF Index: cfe/trunk/tools/clang-scan-deps/CMakeLists.txt =================================================================== --- cfe/trunk/tools/clang-scan-deps/CMakeLists.txt +++ cfe/trunk/tools/clang-scan-deps/CMakeLists.txt @@ -18,6 +18,7 @@ clangParse clangSerialization clangTooling + clangDependencyScanning ) target_link_libraries(clang-scan-deps Index: cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp =================================================================== --- cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp +++ cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp @@ -7,18 +7,11 @@ //===----------------------------------------------------------------------===// #include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/FrontendActions.h" -#include "clang/Frontend/PCHContainerOperations.h" -#include "clang/FrontendTool/Utils.h" #include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Tooling/JSONCompilationDatabase.h" -#include "clang/Tooling/Tooling.h" -#include "llvm/Support/FileSystem.h" #include "llvm/Support/InitLLVM.h" -#include "llvm/Support/JSON.h" #include "llvm/Support/Options.h" -#include "llvm/Support/Path.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" @@ -26,6 +19,7 @@ #include using namespace clang; +using namespace tooling::dependencies; namespace { @@ -43,95 +37,6 @@ raw_ostream &OS; }; -/// Prints out all of the gathered dependencies into one output stream instead -/// of using the output dependency file. -class DependencyPrinter : public DependencyFileGenerator { -public: - DependencyPrinter(std::unique_ptr Opts, - SharedStream &OS) - : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), OS(OS) {} - - void finishedMainFile(DiagnosticsEngine &Diags) override { - OS.applyLocked([this](raw_ostream &OS) { outputDependencyFile(OS); }); - } - -private: - std::unique_ptr Opts; - SharedStream &OS; -}; - -/// A clang tool that runs the preprocessor only for the given compiler -/// invocation. -class PreprocessorOnlyTool : public tooling::ToolAction { -public: - PreprocessorOnlyTool(StringRef WorkingDirectory, SharedStream &OS) - : WorkingDirectory(WorkingDirectory), OS(OS) {} - - bool runInvocation(std::shared_ptr Invocation, - FileManager *FileMgr, - std::shared_ptr PCHContainerOps, - DiagnosticConsumer *DiagConsumer) override { - // Create a compiler instance to handle the actual work. - CompilerInstance Compiler(std::move(PCHContainerOps)); - Compiler.setInvocation(std::move(Invocation)); - FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; - Compiler.setFileManager(FileMgr); - - // Create the compiler's actual diagnostics engine. - Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); - if (!Compiler.hasDiagnostics()) - return false; - - Compiler.createSourceManager(*FileMgr); - - // Create the dependency collector that will collect the produced - // dependencies. - // - // This also moves the existing dependency output options from the - // invocation to the collector. The options in the invocation are reset, - // which ensures that the compiler won't create new dependency collectors, - // and thus won't write out the extra '.d' files to disk. - auto Opts = llvm::make_unique( - std::move(Compiler.getInvocation().getDependencyOutputOpts())); - // We need at least one -MT equivalent for the generator to work. - if (Opts->Targets.empty()) - Opts->Targets = {"clang-scan-deps dependency"}; - Compiler.addDependencyCollector( - std::make_shared(std::move(Opts), OS)); - - auto Action = llvm::make_unique(); - const bool Result = Compiler.ExecuteAction(*Action); - FileMgr->clearStatCache(); - return Result; - } - -private: - StringRef WorkingDirectory; - SharedStream &OS; -}; - -/// A proxy file system that doesn't call `chdir` when changing the working -/// directory of a clang tool. -class ProxyFileSystemWithoutChdir : public llvm::vfs::ProxyFileSystem { -public: - ProxyFileSystemWithoutChdir( - llvm::IntrusiveRefCntPtr FS) - : ProxyFileSystem(std::move(FS)) {} - - llvm::ErrorOr getCurrentWorkingDirectory() const override { - assert(!CWD.empty() && "empty CWD"); - return CWD; - } - - std::error_code setCurrentWorkingDirectory(const Twine &Path) override { - CWD = Path.str(); - return {}; - } - -private: - std::string CWD; -}; - /// The high-level implementation of the dependency discovery tool that runs on /// an individual worker thread. class DependencyScanningTool { @@ -141,31 +46,33 @@ /// \param Compilations The reference to the compilation database that's /// used by the clang tool. DependencyScanningTool(const tooling::CompilationDatabase &Compilations, - SharedStream &OS) - : Compilations(Compilations), OS(OS) { - PCHContainerOps = std::make_shared(); - BaseFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); - } + SharedStream &OS, SharedStream &Errs) + : Compilations(Compilations), OS(OS), Errs(Errs) {} - /// Computes the dependencies for the given file. + /// Computes the dependencies for the given file and prints them out. /// /// \returns True on error. bool runOnFile(const std::string &Input, StringRef CWD) { - BaseFS->setCurrentWorkingDirectory(CWD); - tooling::ClangTool Tool(Compilations, Input, PCHContainerOps, BaseFS); - Tool.clearArgumentsAdjusters(); - Tool.setRestoreWorkingDir(false); - PreprocessorOnlyTool Action(CWD, OS); - return Tool.run(&Action); + auto MaybeFile = Worker.getDependencyFile(Input, CWD, Compilations); + if (!MaybeFile) { + llvm::handleAllErrors( + MaybeFile.takeError(), [this, &Input](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error while scanning dependencies for " << Input << ":\n"; + OS << Err.getMessage(); + }); + }); + return true; + } + OS.applyLocked([&](raw_ostream &OS) { OS << *MaybeFile; }); + return false; } private: + DependencyScanningWorker Worker; const tooling::CompilationDatabase &Compilations; SharedStream &OS; - std::shared_ptr PCHContainerOps; - /// The real filesystem used as a base for all the operations performed by the - /// tool. - llvm::IntrusiveRefCntPtr BaseFS; + SharedStream &Errs; }; llvm::cl::opt Help("h", llvm::cl::desc("Alias for -help"), @@ -225,6 +132,7 @@ return AdjustedArgs; }); + SharedStream Errs(llvm::errs()); // Print out the dependency results to STDOUT by default. SharedStream DependencyOS(llvm::outs()); unsigned NumWorkers = @@ -232,7 +140,7 @@ std::vector> WorkerTools; for (unsigned I = 0; I < NumWorkers; ++I) WorkerTools.push_back(llvm::make_unique( - *AdjustingCompilations, DependencyOS)); + *AdjustingCompilations, DependencyOS, Errs)); std::vector WorkerThreads; std::atomic HadErrors(false);