diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -11,13 +11,69 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/JSONCompilationDatabase.h" +#include "llvm/ADT/StringSet.h" #include namespace clang{ namespace tooling{ namespace dependencies{ +/// The full dependencies and module graph for a specific input. +struct FullDependencies { + /// The name of the C++20 module this translation unit exports. This may + /// include `:` for C++20 module partitons. + /// + /// If the translation unit is not a module then this will be empty. + std::string ExportedModuleName; + + /// The context hash represents the set of compiler options that may make one + /// version of a module incompatible with another. This includes things like + /// language mode, predefined macros, header search paths, etc... + /// + /// Modules with the same name but a different \c ContextHash should be + /// treated as separate modules for the purpose of a build. + std::string ContextHash; + + /// A collection of absolute paths to files that this translation unit + /// directly depends on, not including transitive dependencies. + std::vector FileDeps; + + /// A list of modules this translation unit directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A partial addtional set of command line arguments that can be used to + /// build this translation unit. + /// + /// Call \c getFullAdditionalCommandLine() to get a command line suitable for + /// appending to the original command line to pass to clang. + std::vector AdditionalNonPathCommandLine; + + /// Gets the full addtional command line suitable for appending to the + /// original command line to pass to clang. + /// + /// \param LookupPCMPath this function is called to fill in `-fmodule-file=` + /// flags and for the `-o` flag. It needs to return a + /// path for where the PCM for the given module is to + /// be located. + /// \param LookupModuleDeps this fucntion is called to collect the full + /// transitive set of dependencies for this + /// compilation. + std::vector getAdditionalCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const; +}; + +struct FullDependenciesResult { + FullDependencies FullDeps; + std::vector DiscoveredModules; +}; + /// The high-level implementation of the dependency discovery tool that runs on /// an individual worker thread. class DependencyScanningTool { @@ -35,8 +91,23 @@ getDependencyFile(const tooling::CompilationDatabase &Compilations, StringRef CWD); + /// Collect the full module depenedency graph for the input, ignoring any + /// modules which have already been seen. + /// + /// \param AlreadySeen this is used to not report modules that have previously + /// been reported. Use the same `llvm::StringSet<>` for all + /// calls to `getFullDependencies` for a single + /// `DependencyScanningTool` for a single build. Use a + /// different one for different tools, and clear it between + /// builds. + /// + /// \returns a \c StringError with the diagnostic output if clang errors + /// occurred, \c FullDependencies otherwise. + llvm::Expected + getFullDependencies(const tooling::CompilationDatabase &Compilations, + StringRef CWD, const llvm::StringSet<> &AlreadySeen); + private: - const ScanningOutputFormat Format; DependencyScanningWorker Worker; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -28,16 +28,82 @@ class DependencyConsumer; +/// This is used to refer to a specific module. +/// +/// See \c ModuleDeps for details about what these members mean. +struct ClangModuleDep { + std::string ModuleName; + std::string ContextHash; +}; + struct ModuleDeps { + /// The name of the module. This may include `:` for C++20 module partitons, + /// or a header-name for C++20 header units. std::string ModuleName; - std::string ClangModuleMapFile; - std::string ModulePCMPath; + + /// The context hash of a module represents the set of compiler options that + /// may make one version of a module incompatible with another. This includes + /// things like language mode, predefined macros, header search paths, etc... + /// + /// Modules with the same name but a different \c ContextHash should be + /// treated as separate modules for the purpose of a build. std::string ContextHash; + + /// The path to the modulemap file which defines this module. + /// + /// This can be used to explicitly build this module. This file will + /// additionally appear in \c FileDeps as a dependency. + std::string ClangModuleMapFile; + + /// The path to where an implicit build would put the PCM for this module. + std::string ImplicitModulePCMPath; + + /// A collection of absolute paths to files that this module directly depends + /// on, not including transitive dependencies. llvm::StringSet<> FileDeps; - llvm::StringSet<> ClangModuleDeps; + + /// A list of modules this module directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A partial command line that can be used to build this module. + /// + /// Call \c getFullCommandLine() to get a command line suitable for passing to + /// clang. + std::vector NonPathCommandLine; + + // Used to track which modules that were discovered were directly imported by + // the primary TU. bool ImportedByMainFile = false; + + /// Gets the full command line suitable for passing to clang. + /// + /// \param LookupPCMPath this function is called to fill in `-fmodule-file=` + /// flags and for the `-o` flag. It needs to return a + /// path for where the PCM for the given module is to + /// be located. + /// \param LookupModuleDeps this fucntion is called to collect the full + /// transitive set of dependencies for this + /// compilation. + std::vector getFullCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const; }; +namespace detail { +/// Append the `-fmodule-file=` and `-fmodule-map-file=` arguments for the +/// modules in \c Modules transitively, along with other needed arguments to +/// use explicitly built modules. +void appendCommonModuleArguments( + llvm::ArrayRef Modules, + std::function LookupPCMPath, + std::function LookupModuleDeps, + std::vector &Result); +} // namespace detail + class ModuleDepCollector; class ModuleDepCollectorPP final : public PPCallbacks { @@ -54,6 +120,8 @@ StringRef SearchPath, StringRef RelativePath, const Module *Imported, SrcMgr::CharacteristicKind FileType) override; + void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path, + const Module *Imported) override; void EndOfMainFile() override; @@ -62,16 +130,18 @@ ModuleDepCollector &MDC; llvm::DenseSet DirectDeps; + void handleImport(const Module *Imported); void handleTopLevelModule(const Module *M); - void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD); - void addModuleDep(const Module *M, ModuleDeps &MD); - - void addDirectDependencies(const Module *Mod); + void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules); + void addModuleDep(const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules); }; class ModuleDepCollector final : public DependencyCollector { public: - ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C); + ModuleDepCollector(std::unique_ptr Opts, + CompilerInstance &I, DependencyConsumer &C); void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; @@ -85,6 +155,7 @@ std::string ContextHash; std::vector MainDeps; std::unordered_map Deps; + std::unique_ptr Opts; }; } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -8,24 +8,25 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Frontend/Utils.h" -#include "llvm/Support/JSON.h" - -static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { - std::vector Strings; - for (auto &&I : Set) - Strings.push_back(I.getKey()); - std::sort(Strings.begin(), Strings.end()); - return llvm::json::Array(Strings); -} namespace clang{ namespace tooling{ namespace dependencies{ +std::vector FullDependencies::getAdditionalCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const { + std::vector Ret = AdditionalNonPathCommandLine; + + dependencies::detail::appendCommonModuleArguments( + ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret); + + return Ret; +} + DependencyScanningTool::DependencyScanningTool( DependencyScanningService &Service) - : Format(Service.getFormat()), Worker(Service) { -} + : Worker(Service) {} llvm::Expected DependencyScanningTool::getDependencyFile( const tooling::CompilationDatabase &Compilations, StringRef CWD) { @@ -75,8 +76,33 @@ std::vector Dependencies; }; + // We expect a single command here because if a source file occurs multiple + // times in the original CDB, then `computeDependencies` would run the + // `DependencyScanningAction` once for every time the input occured in the + // CDB. Instead we split up the CDB into single command chunks to avoid this + // behavior. + assert(Compilations.getAllCompileCommands().size() == 1 && + "Expected a compilation database with a single command!"); + std::string Input = Compilations.getAllCompileCommands().front().Filename; + + MakeDependencyPrinterConsumer Consumer; + auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +llvm::Expected +DependencyScanningTool::getFullDependencies( + const tooling::CompilationDatabase &Compilations, StringRef CWD, + const llvm::StringSet<> &AlreadySeen) { class FullDependencyPrinterConsumer : public DependencyConsumer { public: + FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + void handleFileDependency(const DependencyOutputOptions &Opts, StringRef File) override { Dependencies.push_back(File); @@ -90,55 +116,41 @@ ContextHash = std::move(Hash); } - void printDependencies(std::string &S, StringRef MainFile) { - // Sort the modules by name to get a deterministic order. - std::vector Modules; - for (auto &&Dep : ClangModuleDeps) - Modules.push_back(Dep.first); - std::sort(Modules.begin(), Modules.end()); + FullDependenciesResult getFullDependencies() const { + FullDependencies FD; - llvm::raw_string_ostream OS(S); + FD.ContextHash = std::move(ContextHash); - using namespace llvm::json; + FD.FileDeps.assign(Dependencies.begin(), Dependencies.end()); - Array Imports; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; if (MD.ImportedByMainFile) - Imports.push_back(MD.ModuleName); + FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); } - Array Mods; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; - Object Mod{ - {"name", MD.ModuleName}, - {"file-deps", toJSONSorted(MD.FileDeps)}, - {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, - {"clang-modulemap-file", MD.ClangModuleMapFile}, - }; - Mods.push_back(std::move(Mod)); - } + FullDependenciesResult FDR; - Object O{ - {"input-file", MainFile}, - {"clang-context-hash", ContextHash}, - {"file-deps", Dependencies}, - {"clang-module-deps", std::move(Imports)}, - {"clang-modules", std::move(Mods)}, - }; + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FDR.DiscoveredModules.push_back(std::move(M.second)); + } - S = llvm::formatv("{0:2},\n", Value(std::move(O))).str(); - return; + FDR.FullDeps = std::move(FD); + return FDR; } private: std::vector Dependencies; std::unordered_map ClangModuleDeps; std::string ContextHash; + std::vector OutputPaths; + const llvm::StringSet<> &AlreadySeen; }; - // We expect a single command here because if a source file occurs multiple // times in the original CDB, then `computeDependencies` would run the // `DependencyScanningAction` once for every time the input occured in the @@ -147,26 +159,13 @@ assert(Compilations.getAllCompileCommands().size() == 1 && "Expected a compilation database with a single command!"); std::string Input = Compilations.getAllCompileCommands().front().Filename; - - if (Format == ScanningOutputFormat::Make) { - MakeDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output); - return Output; - } else { - FullDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output, Input); - return Output; - } + + FullDependencyPrinterConsumer Consumer(AlreadySeen); + llvm::Error Result = + Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + return Consumer.getFullDependencies(); } } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -142,11 +142,18 @@ Consumer)); break; case ScanningOutputFormat::Full: - Compiler.addDependencyCollector( - std::make_shared(Compiler, Consumer)); + Compiler.addDependencyCollector(std::make_shared( + std::move(Opts), Compiler, Consumer)); break; } + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing and header search pruning to reduce + // the impact of strict context hashing. + Compiler.getHeaderSearchOpts().ModulesStrictContextHash = false; + Consumer.handleContextHash(Compiler.getInvocation().getModuleHash()); auto Action = std::make_unique(); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -17,6 +17,47 @@ using namespace tooling; using namespace dependencies; +std::vector ModuleDeps::getFullCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const { + std::vector Ret = NonPathCommandLine; + + // TODO: Build full command line. That also means capturing the original + // command line into NonPathCommandLine. + + dependencies::detail::appendCommonModuleArguments( + ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret); + + return Ret; +} + +void dependencies::detail::appendCommonModuleArguments( + llvm::ArrayRef Modules, + std::function LookupPCMPath, + std::function LookupModuleDeps, + std::vector &Result) { + llvm::StringSet<> AlreadyAdded; + + std::function)> AddArgs = + [&](llvm::ArrayRef Modules) { + for (const ClangModuleDep &CMD : Modules) { + if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second) + continue; + const ModuleDeps &M = LookupModuleDeps(CMD); + // Depth first traversal. + AddArgs(M.ClangModuleDeps); + Result.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str()); + if (!M.ClangModuleMapFile.empty()) { + Result.push_back("-fmodule-map-file=" + M.ClangModuleMapFile); + } + } + }; + + Result.push_back("-fno-implicit-modules"); + Result.push_back("-fno-implicit-module-maps"); + AddArgs(Modules); +} + void ModuleDepCollectorPP::FileChanged(SourceLocation Loc, FileChangeReason Reason, SrcMgr::CharacteristicKind FileType, @@ -50,7 +91,16 @@ // here as `FileChanged` will never see it. MDC.MainDeps.push_back(FileName); } + handleImport(Imported); +} +void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc, + ModuleIdPath Path, + const Module *Imported) { + handleImport(Imported); +} + +void ModuleDepCollectorPP::handleImport(const Module *Imported) { if (!Imported) return; @@ -71,9 +121,8 @@ for (auto &&I : MDC.Deps) MDC.Consumer.handleModuleDependency(I.second); - DependencyOutputOptions Opts; for (auto &&I : MDC.MainDeps) - MDC.Consumer.handleFileDependency(Opts, I); + MDC.Consumer.handleFileDependency(*MDC.Opts, I); } void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { @@ -94,7 +143,7 @@ MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : ""; MD.ModuleName = M->getFullModuleName(); - MD.ModulePCMPath = M->getASTFile()->getName(); + MD.ImplicitModulePCMPath = M->getASTFile()->getName(); MD.ContextHash = MDC.ContextHash; serialization::ModuleFile *MF = MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile()); @@ -103,30 +152,38 @@ MD.FileDeps.insert(IF.getFile()->getName()); }); - addAllSubmoduleDeps(M, MD); + llvm::DenseSet AddedModules; + addAllSubmoduleDeps(M, MD, AddedModules); } -void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M, - ModuleDeps &MD) { - addModuleDep(M, MD); +void ModuleDepCollectorPP::addAllSubmoduleDeps( + const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules) { + addModuleDep(M, MD, AddedModules); for (const Module *SubM : M->submodules()) - addAllSubmoduleDeps(SubM, MD); + addAllSubmoduleDeps(SubM, MD, AddedModules); } -void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) { +void ModuleDepCollectorPP::addModuleDep( + const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules) { for (const Module *Import : M->Imports) { if (Import->getTopLevelModule() != M->getTopLevelModule()) { - MD.ClangModuleDeps.insert(Import->getTopLevelModuleName()); + if (AddedModules.insert(Import->getTopLevelModule()).second) + MD.ClangModuleDeps.push_back( + {Import->getTopLevelModuleName(), + Instance.getInvocation().getModuleHash()}); handleTopLevelModule(Import->getTopLevelModule()); } } } -ModuleDepCollector::ModuleDepCollector(CompilerInstance &I, - DependencyConsumer &C) - : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) { -} +ModuleDepCollector::ModuleDepCollector( + std::unique_ptr Opts, CompilerInstance &I, + DependencyConsumer &C) + : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()), + Opts(std::move(Opts)) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { PP.addPPCallbacks(std::make_unique(Instance, *this)); diff --git a/clang/test/ClangScanDeps/Inputs/modules_cdb.json b/clang/test/ClangScanDeps/Inputs/modules_cdb.json --- a/clang/test/ClangScanDeps/Inputs/modules_cdb.json +++ b/clang/test/ClangScanDeps/Inputs/modules_cdb.json @@ -1,13 +1,22 @@ [ { "directory": "DIR", - "command": "clang -E -fsyntax-only DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "command": "clang -E DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input2.cpp" }, { "directory": "DIR", "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o a.o", + "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o b.o", + "file": "DIR/modules_cdb_input.cpp" } ] - diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp --- a/clang/test/ClangScanDeps/modules-full.cpp +++ b/clang/test/ClangScanDeps/modules-full.cpp @@ -1,6 +1,5 @@ // RUN: rm -rf %t.dir // RUN: rm -rf %t.cdb -// RUN: rm -rf %t.module-cache // RUN: mkdir -p %t.dir // RUN: cp %s %t.dir/modules_cdb_input.cpp // RUN: cp %s %t.dir/modules_cdb_input2.cpp @@ -11,67 +10,146 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb // // RUN: echo %t.dir > %t.result -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -full-command-line \ // RUN: -mode preprocess-minimized-sources -format experimental-full >> %t.result -// RUN: cat %t.result | FileCheck --check-prefixes=CHECK %s +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK %s // FIXME: Backslash issues. // XFAIL: system-windows #include "header.h" -// CHECK: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]] +// CHECK: [[PREFIX:.*]] +// CHECK-NEXT: { +// CHECK-NEXT: "modules": [ // CHECK-NEXT: { -// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH:[A-Z0-9]+]]", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header2" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: }, -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header2.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header2" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT:}, -// CHECK-NEXT:{ -// CHECK-NOT: "clang-context-hash": "[[CONTEXT_HASH]]", -// CHECK-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT:}, +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1:[A-Z0-9]+]]", +// CHECK-NEXT: "module-name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2:[A-Z0-9]+]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header2.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -15,6 +15,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileUtilities.h" #include "llvm/Support/InitLLVM.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" @@ -129,6 +130,11 @@ llvm::cl::init(ScanningOutputFormat::Make), llvm::cl::cat(DependencyScannerCategory)); +static llvm::cl::opt FullCommandLine( + "full-command-line", + llvm::cl::desc("Include the full command lines to use to build modules"), + llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory)); + llvm::cl::opt NumThreads("j", llvm::cl::Optional, llvm::cl::desc("Number of worker threads to use (default: use " @@ -189,9 +195,10 @@ /// based on the result. /// /// \returns True on error. -static bool handleDependencyToolResult(const std::string &Input, - llvm::Expected &MaybeFile, - SharedStream &OS, SharedStream &Errs) { +static bool +handleMakeDependencyToolResult(const std::string &Input, + llvm::Expected &MaybeFile, + SharedStream &OS, SharedStream &Errs) { if (!MaybeFile) { llvm::handleAllErrors( MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) { @@ -206,6 +213,184 @@ return false; } +static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { + std::vector Strings; + for (auto &&I : Set) + Strings.push_back(I.getKey()); + std::sort(Strings.begin(), Strings.end()); + return llvm::json::Array(Strings); +} + +static llvm::json::Array toJSONSorted(std::vector V) { + std::sort(V.begin(), V.end(), + [](const ClangModuleDep &A, const ClangModuleDep &B) { + return std::tie(A.ModuleName, A.ContextHash) < + std::tie(B.ModuleName, B.ContextHash); + }); + + llvm::json::Array Ret; + for (const ClangModuleDep &CMD : V) + Ret.push_back(llvm::json::Object( + {{"module-name", CMD.ModuleName}, {"context-hash", CMD.ContextHash}})); + return Ret; +} + +// Thread safe. +class FullDeps { +public: + void mergeDeps(StringRef Input, FullDependenciesResult FDR, + size_t InputIndex) { + const FullDependencies &FD = FDR.FullDeps; + + InputDeps ID; + ID.FileName = Input; + ID.ContextHash = std::move(FD.ContextHash); + ID.FileDeps = std::move(FD.FileDeps); + ID.ModuleDeps = std::move(FD.ClangModuleDeps); + + std::unique_lock ul(Lock); + for (const ModuleDeps &MD : FDR.DiscoveredModules) { + auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0}); + if (I != Modules.end()) { + I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); + continue; + } + Modules.insert( + I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)}); + } + + if (FullCommandLine) + ID.AdditonalCommandLine = FD.getAdditionalCommandLine( + [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, + [&](ClangModuleDep CMD) -> const ModuleDeps & { + return lookupModuleDeps(CMD); + }); + + Inputs.push_back(std::move(ID)); + } + + void printFullOutput(raw_ostream &OS) { + // Sort the modules by name to get a deterministic order. + std::vector ModuleNames; + for (auto &&M : Modules) + ModuleNames.push_back(M.first); + std::sort(ModuleNames.begin(), ModuleNames.end(), + [](const ContextModulePair &A, const ContextModulePair &B) { + return std::tie(A.ModuleName, A.InputIndex) < + std::tie(B.ModuleName, B.InputIndex); + }); + + std::sort(Inputs.begin(), Inputs.end(), + [](const InputDeps &A, const InputDeps &B) { + return A.FileName < B.FileName; + }); + + using namespace llvm::json; + + Array OutModules; + for (auto &&ModName : ModuleNames) { + auto &MD = Modules[ModName]; + Object O{ + {"name", MD.ModuleName}, + {"context-hash", MD.ContextHash}, + {"file-deps", toJSONSorted(MD.FileDeps)}, + {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, + {"clang-modulemap-file", MD.ClangModuleMapFile}, + {"command-line", + FullCommandLine + ? MD.getFullCommandLine( + [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, + [&](ClangModuleDep CMD) -> const ModuleDeps & { + return lookupModuleDeps(CMD); + }) + : MD.NonPathCommandLine}, + }; + OutModules.push_back(std::move(O)); + } + + Array TUs; + for (auto &&I : Inputs) { + Object O{ + {"input-file", I.FileName}, + {"clang-context-hash", I.ContextHash}, + {"file-deps", I.FileDeps}, + {"clang-module-deps", toJSONSorted(I.ModuleDeps)}, + {"command-line", I.AdditonalCommandLine}, + }; + TUs.push_back(std::move(O)); + } + + Object Output{ + {"modules", std::move(OutModules)}, + {"translation-units", std::move(TUs)}, + }; + + OS << llvm::formatv("{0:2}\n", Value(std::move(Output))); + } + +private: + StringRef lookupPCMPath(ClangModuleDep CMD) { + return Modules[ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}] + .ImplicitModulePCMPath; + } + + const ModuleDeps &lookupModuleDeps(ClangModuleDep CMD) { + auto I = + Modules.find(ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}); + assert(I != Modules.end()); + return I->second; + }; + + struct ContextModulePair { + std::string ContextHash; + std::string ModuleName; + mutable size_t InputIndex; + + bool operator==(const ContextModulePair &Other) const { + return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName; + } + }; + + struct ContextModulePairHasher { + std::size_t operator()(const ContextModulePair &CMP) const { + using llvm::hash_combine; + + return hash_combine(CMP.ContextHash, CMP.ModuleName); + } + }; + + struct InputDeps { + std::string FileName; + std::string ContextHash; + std::vector FileDeps; + std::vector ModuleDeps; + std::vector AdditonalCommandLine; + }; + + std::mutex Lock; + std::unordered_map + Modules; + std::vector Inputs; +}; + +static bool handleFullDependencyToolResult( + const std::string &Input, + llvm::Expected &MaybeFullDeps, FullDeps &FD, + size_t InputIndex, SharedStream &OS, SharedStream &Errs) { + if (!MaybeFullDeps) { + llvm::handleAllErrors( + MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error while scanning dependencies for " << Input << ":\n"; + OS << Err.getMessage(); + }); + }); + return true; + } + FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex); + return false; +} + int main(int argc, const char **argv) { llvm::InitLLVM X(argc, argv); llvm::cl::HideUnrelatedOptions(DependencyScannerCategory); @@ -316,6 +501,7 @@ std::vector WorkerThreads; std::atomic HadErrors(false); + FullDeps FD; std::mutex Lock; size_t Index = 0; @@ -324,26 +510,38 @@ << " files using " << NumWorkers << " workers\n"; } for (unsigned I = 0; I < NumWorkers; ++I) { - auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools, + auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { + llvm::StringSet<> AlreadySeenModules; while (true) { const SingleCommandCompilationDatabase *Input; std::string Filename; std::string CWD; + size_t LocalIndex; // Take the next input. { std::unique_lock LockGuard(Lock); if (Index >= Inputs.size()) return; + LocalIndex = Index; Input = &Inputs[Index++]; tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0]; Filename = std::move(Cmd.Filename); CWD = std::move(Cmd.Directory); } // Run the tool on it. - auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); - if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs)) - HadErrors = true; + if (Format == ScanningOutputFormat::Make) { + auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); + if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, + Errs)) + HadErrors = true; + } else { + auto MaybeFullDeps = WorkerTools[I]->getFullDependencies( + *Input, CWD, AlreadySeenModules); + if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } } }; #if LLVM_ENABLE_THREADS @@ -356,5 +554,8 @@ for (auto &W : WorkerThreads) W.join(); + if (Format == ScanningOutputFormat::Full) + FD.printFullOutput(llvm::outs()); + return HadErrors; }