diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -12,12 +12,28 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Tooling/JSONCompilationDatabase.h" +#include "llvm/ADT/StringSet.h" #include namespace clang{ namespace tooling{ namespace dependencies{ +/// The full dependencies and module graph for a specific input. +struct FullDependencies { + /// The context in which these modules are valid. + std::string ContextHash; + /// The output paths the compiler would write to if this was a -c invocation. + std::vector OutputPaths; + /// Files that the input directly depends on. + std::vector DirectFileDependencies; + /// Modules that the input directly imports. + std::vector DirectModuleDependencies; + /// The Clang modules this input transitively depends on that have not already + /// been reported. + std::vector ClangModuleDeps; +}; + /// The high-level implementation of the dependency discovery tool that runs on /// an individual worker thread. class DependencyScanningTool { @@ -38,8 +54,16 @@ getDependencyFile(const tooling::CompilationDatabase &Compilations, StringRef CWD); + /// Collect the full module depenedency graph for the input, ignoring any + /// modules which have already been seen. + /// + /// \returns a \c StringError with the diagnostic output if clang errors + /// occurred, \c FullDependencies otherwise. + llvm::Expected + getFullDependencies(const tooling::CompilationDatabase &Compilations, + StringRef CWD, const llvm::StringSet<> &AlreadySeen); + private: - const ScanningOutputFormat Format; DependencyScanningWorker Worker; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -71,7 +71,8 @@ class ModuleDepCollector final : public DependencyCollector { public: - ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C); + ModuleDepCollector(std::unique_ptr Opts, + CompilerInstance &I, DependencyConsumer &C); void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; @@ -85,6 +86,7 @@ std::string ContextHash; std::vector MainDeps; std::unordered_map Deps; + std::unique_ptr Opts; }; } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -8,15 +8,6 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Frontend/Utils.h" -#include "llvm/Support/JSON.h" - -static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { - std::vector Strings; - for (auto &&I : Set) - Strings.push_back(I.getKey()); - std::sort(Strings.begin(), Strings.end()); - return llvm::json::Array(Strings); -} namespace clang{ namespace tooling{ @@ -24,8 +15,7 @@ DependencyScanningTool::DependencyScanningTool( DependencyScanningService &Service) - : Format(Service.getFormat()), Worker(Service) { -} + : Worker(Service) {} llvm::Expected DependencyScanningTool::getDependencyFile( const tooling::CompilationDatabase &Compilations, StringRef CWD) { @@ -75,10 +65,36 @@ std::vector Dependencies; }; + // We expect a single command here because if a source file occurs multiple + // times in the original CDB, then `computeDependencies` would run the + // `DependencyScanningAction` once for every time the input occured in the + // CDB. Instead we split up the CDB into single command chunks to avoid this + // behavior. + assert(Compilations.getAllCompileCommands().size() == 1 && + "Expected a compilation database with a single command!"); + std::string Input = Compilations.getAllCompileCommands().front().Filename; + + MakeDependencyPrinterConsumer Consumer; + auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +llvm::Expected DependencyScanningTool::getFullDependencies( + const tooling::CompilationDatabase &Compilations, StringRef CWD, + const llvm::StringSet<> &AlreadySeen) { class FullDependencyPrinterConsumer : public DependencyConsumer { public: + FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + void handleFileDependency(const DependencyOutputOptions &Opts, StringRef File) override { + if (OutputPaths.empty()) + OutputPaths = Opts.Targets; Dependencies.push_back(File); } @@ -90,55 +106,41 @@ ContextHash = std::move(Hash); } - void printDependencies(std::string &S, StringRef MainFile) { - // Sort the modules by name to get a deterministic order. - std::vector Modules; - for (auto &&Dep : ClangModuleDeps) - Modules.push_back(Dep.first); - std::sort(Modules.begin(), Modules.end()); + FullDependencies getFullDependencies() const { + FullDependencies FD; - llvm::raw_string_ostream OS(S); + FD.ContextHash = std::move(ContextHash); - using namespace llvm::json; + FD.DirectFileDependencies.assign(Dependencies.begin(), + Dependencies.end()); - Array Imports; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; + FD.OutputPaths = std::move(OutputPaths); + + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; if (MD.ImportedByMainFile) - Imports.push_back(MD.ModuleName); + FD.DirectModuleDependencies.push_back(MD.ModuleName); } - Array Mods; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; - Object Mod{ - {"name", MD.ModuleName}, - {"file-deps", toJSONSorted(MD.FileDeps)}, - {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, - {"clang-modulemap-file", MD.ClangModuleMapFile}, - }; - Mods.push_back(std::move(Mod)); + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FD.ClangModuleDeps.push_back(std::move(M.second)); } - Object O{ - {"input-file", MainFile}, - {"clang-context-hash", ContextHash}, - {"file-deps", Dependencies}, - {"clang-module-deps", std::move(Imports)}, - {"clang-modules", std::move(Mods)}, - }; - - S = llvm::formatv("{0:2},\n", Value(std::move(O))).str(); - return; + return FD; } private: std::vector Dependencies; std::unordered_map ClangModuleDeps; std::string ContextHash; + std::vector OutputPaths; + const llvm::StringSet<> &AlreadySeen; }; - // We expect a single command here because if a source file occurs multiple // times in the original CDB, then `computeDependencies` would run the // `DependencyScanningAction` once for every time the input occured in the @@ -147,26 +149,13 @@ assert(Compilations.getAllCompileCommands().size() == 1 && "Expected a compilation database with a single command!"); std::string Input = Compilations.getAllCompileCommands().front().Filename; - - if (Format == ScanningOutputFormat::Make) { - MakeDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output); - return Output; - } else { - FullDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output, Input); - return Output; - } + + FullDependencyPrinterConsumer Consumer(AlreadySeen); + llvm::Error Result = + Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + return Consumer.getFullDependencies(); } } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -142,11 +142,18 @@ Consumer)); break; case ScanningOutputFormat::Full: - Compiler.addDependencyCollector( - std::make_shared(Compiler, Consumer)); + Compiler.addDependencyCollector(std::make_shared( + std::move(Opts), Compiler, Consumer)); break; } + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing and header search pruning to reduce + // the impact of strict context hashing. + Compiler.getHeaderSearchOpts().ModulesStrictContextHash = false; + Consumer.handleContextHash(Compiler.getInvocation().getModuleHash()); auto Action = std::make_unique(); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -71,9 +71,8 @@ for (auto &&I : MDC.Deps) MDC.Consumer.handleModuleDependency(I.second); - DependencyOutputOptions Opts; for (auto &&I : MDC.MainDeps) - MDC.Consumer.handleFileDependency(Opts, I); + MDC.Consumer.handleFileDependency(*MDC.Opts, I); } void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { @@ -124,10 +123,11 @@ } } -ModuleDepCollector::ModuleDepCollector(CompilerInstance &I, - DependencyConsumer &C) - : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) { -} +ModuleDepCollector::ModuleDepCollector( + std::unique_ptr Opts, CompilerInstance &I, + DependencyConsumer &C) + : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()), + Opts(std::move(Opts)) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { PP.addPPCallbacks(std::make_unique(Instance, *this)); diff --git a/clang/test/ClangScanDeps/Inputs/modules_cdb.json b/clang/test/ClangScanDeps/Inputs/modules_cdb.json --- a/clang/test/ClangScanDeps/Inputs/modules_cdb.json +++ b/clang/test/ClangScanDeps/Inputs/modules_cdb.json @@ -1,13 +1,22 @@ [ { "directory": "DIR", - "command": "clang -E -fsyntax-only DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "command": "clang -E DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input2.cpp" }, { "directory": "DIR", "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o a.o", + "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o b.o", + "file": "DIR/modules_cdb_input.cpp" } ] - diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp --- a/clang/test/ClangScanDeps/modules-full.cpp +++ b/clang/test/ClangScanDeps/modules-full.cpp @@ -11,67 +11,103 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb // // RUN: echo %t.dir > %t.result -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 \ // RUN: -mode preprocess-minimized-sources -format experimental-full >> %t.result -// RUN: cat %t.result | FileCheck --check-prefixes=CHECK %s +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK %s // FIXME: Backslash issues. // XFAIL: system-windows #include "header.h" -// CHECK: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]] +// CHECK: [[PREFIX:.*]] +// CHECK-NEXT: { +// CHECK-NEXT: "modules": [ // CHECK-NEXT: { -// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH:[A-Z0-9]+]]", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header2" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: }, -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header2.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header2" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT:}, -// CHECK-NEXT:{ -// CHECK-NOT: "clang-context-hash": "[[CONTEXT_HASH]]", -// CHECK-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT:}, +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: "header2" +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1:[A-Z0-9]+]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2:[A-Z0-9]+]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header2.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: "header1" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp", +// CHECK-NEXT: "output-files": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.o" +// CHECK-NEXT: ] +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: "header1" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp", +// CHECK-NEXT: "output-files": [ +// CHECK-NEXT: "a.o" +// CHECK-NEXT: ] +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: "header1" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp", +// CHECK-NEXT: "output-files": [ +// CHECK-NEXT: "b.o" +// CHECK-NEXT: ] +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: "header1" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input2.cpp", +// CHECK-NEXT: "output-files": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -13,6 +13,7 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Tooling/JSONCompilationDatabase.h" #include "llvm/Support/InitLLVM.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/Options.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" @@ -130,9 +131,10 @@ /// based on the result. /// /// \returns True on error. -static bool handleDependencyToolResult(const std::string &Input, - llvm::Expected &MaybeFile, - SharedStream &OS, SharedStream &Errs) { +static bool +handleMakeDependencyToolResult(const std::string &Input, + llvm::Expected &MaybeFile, + SharedStream &OS, SharedStream &Errs) { if (!MaybeFile) { llvm::handleAllErrors( MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) { @@ -147,6 +149,141 @@ return false; } +static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { + std::vector Strings; + for (auto &&I : Set) + Strings.push_back(I.getKey()); + std::sort(Strings.begin(), Strings.end()); + return llvm::json::Array(Strings); +} + +// Thread safe. +class FullDeps { +public: + void mergeDeps(StringRef Input, FullDependencies FD, size_t InputIndex) { + InputDeps ID; + ID.FileName = Input; + ID.ContextHash = std::move(FD.ContextHash); + ID.FileDeps = std::move(FD.DirectFileDependencies); + ID.ModuleDeps = std::move(FD.DirectModuleDependencies); + ID.OutputPaths = std::move(FD.OutputPaths); + + std::unique_lock ul(Lock); + for (ModuleDeps &MD : FD.ClangModuleDeps) { + auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0}); + if (I != Modules.end()) { + I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); + continue; + } + Modules.insert( + I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)}); + } + + Inputs.push_back(std::move(ID)); + } + + void printFullOutput(raw_ostream &OS) { + // Sort the modules by name to get a deterministic order. + std::vector ModuleNames; + for (auto &&M : Modules) + ModuleNames.push_back(M.first); + std::sort(ModuleNames.begin(), ModuleNames.end(), + [](const ContextModulePair &A, const ContextModulePair &B) { + return std::tie(A.ModuleName, A.InputIndex) < + std::tie(B.ModuleName, B.InputIndex); + }); + + std::sort(Inputs.begin(), Inputs.end(), + [](const InputDeps &A, const InputDeps &B) { + return std::tie(A.FileName, A.OutputPaths) < + std::tie(B.FileName, B.OutputPaths); + }); + + using namespace llvm::json; + + Array OutModules; + for (auto &&ModName : ModuleNames) { + auto &MD = Modules[ModName]; + Object O{ + {"name", MD.ModuleName}, + {"context-hash", MD.ContextHash}, + {"file-deps", toJSONSorted(MD.FileDeps)}, + {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, + {"clang-modulemap-file", MD.ClangModuleMapFile}, + }; + OutModules.push_back(std::move(O)); + } + + Array TUs; + for (auto &&I : Inputs) { + Object O{ + {"input-file", I.FileName}, + {"clang-context-hash", I.ContextHash}, + {"file-deps", I.FileDeps}, + {"clang-module-deps", I.ModuleDeps}, + {"output-files", I.OutputPaths}, + }; + TUs.push_back(std::move(O)); + } + + Object Output{ + {"modules", std::move(OutModules)}, + {"translation-units", std::move(TUs)}, + }; + + OS << llvm::formatv("{0:2}\n", Value(std::move(Output))); + } + +private: + struct ContextModulePair { + std::string ContextHash; + std::string ModuleName; + mutable size_t InputIndex; + + bool operator==(const ContextModulePair &Other) const { + return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName; + } + }; + + struct ContextModulePairHasher { + std::size_t operator()(const ContextModulePair &CMP) const { + using llvm::hash_combine; + + return hash_combine(CMP.ContextHash, CMP.ModuleName); + } + }; + + struct InputDeps { + std::string FileName; + std::string ContextHash; + std::vector FileDeps; + std::vector ModuleDeps; + std::vector OutputPaths; + }; + + std::mutex Lock; + std::unordered_map + Modules; + std::vector Inputs; +}; + +static bool handleFullDependencyToolResult( + const std::string &Input, llvm::Expected &MaybeFullDeps, + FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) { + if (!MaybeFullDeps) { + llvm::handleAllErrors( + MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error while scanning dependencies for " << Input << ":\n"; + OS << Err.getMessage(); + }); + }); + return true; + } + FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex); + return false; +} + int main(int argc, const char **argv) { llvm::InitLLVM X(argc, argv); llvm::cl::HideUnrelatedOptions(DependencyScannerCategory); @@ -243,6 +380,7 @@ std::vector WorkerThreads; std::atomic HadErrors(false); + FullDeps FD; std::mutex Lock; size_t Index = 0; @@ -251,26 +389,38 @@ << " files using " << NumWorkers << " workers\n"; } for (unsigned I = 0; I < NumWorkers; ++I) { - auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools, + auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { + llvm::StringSet<> AlreadySeenModules; while (true) { const SingleCommandCompilationDatabase *Input; std::string Filename; std::string CWD; + size_t LocalIndex; // Take the next input. { std::unique_lock LockGuard(Lock); if (Index >= Inputs.size()) return; + LocalIndex = Index; Input = &Inputs[Index++]; tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0]; Filename = std::move(Cmd.Filename); CWD = std::move(Cmd.Directory); } // Run the tool on it. - auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); - if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs)) - HadErrors = true; + if (Format == ScanningOutputFormat::Make) { + auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); + if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, + Errs)) + HadErrors = true; + } else { + auto MaybeFullDeps = WorkerTools[I]->getFullDependencies( + *Input, CWD, AlreadySeenModules); + if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } } }; #if LLVM_ENABLE_THREADS @@ -283,5 +433,8 @@ for (auto &W : WorkerThreads) W.join(); + if (Format == ScanningOutputFormat::Full) + FD.printFullOutput(llvm::outs()); + return HadErrors; }