[clang][ScanDeps] Fix issue with multiple commands with the same input.

[Bigcheese]

Summary

Previously, given a CompilationDatabase with two commands for the same
source file we would report that file twice with the union of the
dependencies for each command both times.

This was due to the way ClangTool runs actions given an input source
file (see the comment in DependencyScanningTool.cpp). This commit adds
a SingleCommandCompilationDatabase that is created with each
CompileCommand in the original CDB, which is then used for each
ClangTool invocation. This gives us a single run of
DependencyScanningAction per CompileCommand.

I looked at using AllTUsToolExecutor which is a parallel tool
executor, but I'm not sure it's suitable for clang-scan-deps as it
does a lot more sharing of state than AllTUsToolExecutor expects.

I've added Manuel as a reviewer to see if there's an obviously better way to do this with libTooling.

[arphaman]

LGTM, Thanks

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

 /// The high-level implementation of the dependency discovery tool that runs on
 /// an individual worker thread.
 class DependencyScanningTool {
 public:
   /// Construct a dependency scanning tool.
   ///
   /// \param Compilations     The reference to the compilation database that's
   /// used by the clang tool.
-  DependencyScanningTool(
-      DependencyScanningService &Service,
-      const clang::tooling::CompilationDatabase &Compilations);
+  DependencyScanningTool(DependencyScanningService &Service);
 
   /// Print out the dependency information into a string using the dependency
   /// file format that is specified in the options (-MD is the default) and
   /// return it.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
   /// occurred, dependency file contents otherwise.
-  llvm::Expected<std::string> getDependencyFile(const std::string &Input,
-                                                StringRef CWD);
+  llvm::Expected<std::string>
+  getDependencyFile(const tooling::CompilationDatabase &Compilations,
 
 private:
   const ScanningOutputFormat Format;
   DependencyScanningWorker Worker;
-  const tooling::CompilationDatabase &Compilations;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_TOOL_H

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
   return llvm::json::Array(Strings);
 }
 
 namespace clang{
 namespace tooling{
 namespace dependencies{
 
 DependencyScanningTool::DependencyScanningTool(
-    DependencyScanningService &Service,
-    const tooling::CompilationDatabase &Compilations)
-    : Format(Service.getFormat()), Worker(Service), Compilations(Compilations) {
+    DependencyScanningService &Service)
+    : Format(Service.getFormat()), Worker(Service) {
 }
 
-llvm::Expected<std::string>
-DependencyScanningTool::getDependencyFile(const std::string &Input,
-                                          StringRef CWD) {
+llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
+    const tooling::CompilationDatabase &Compilations, StringRef CWD) {
   /// Prints out all of the gathered dependencies into a string.
   class MakeDependencyPrinterConsumer : public DependencyConsumer {
   public:
     void handleFileDependency(const DependencyOutputOptions &Opts,
                               StringRef File) override {
       if (!this->Opts)
         this->Opts = std::make_unique<DependencyOutputOptions>(Opts);
       Dependencies.push_back(File);
public:
     }
 
   private:
     std::vector<std::string> Dependencies;
     std::unordered_map<std::string, ModuleDeps> ClangModuleDeps;
     std::string ContextHash;
   };
 
+  
+  // We expect a single command here because if a source file occurs multiple
+  // times in the original CDB, then `computeDependencies` would run the
+  // `DependencyScanningAction` once for every time the input occured in the
+  // CDB. Instead we split up the CDB into single command chunks to avoid this
+  // behavior.
+  assert(Compilations.getAllCompileCommands().size() == 1 &&
+         "Expected a compilation database with a single command!");
+  std::string Input = Compilations.getAllCompileCommands().front().Filename;
+  
   if (Format == ScanningOutputFormat::Make) {
     MakeDependencyPrinterConsumer Consumer;
     auto Result =
         Worker.computeDependencies(Input, CWD, Compilations, Consumer);
     if (Result)
       return std::move(Result);
     std::string Output;
     Consumer.printDependencies(Output);

clang/test/ClangScanDeps/Inputs/regular_cdb.json

 [
 {
   "directory": "DIR",
   "command": "clang -E -fsyntax-only DIR/regular_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/regular_cdb2.d",
   "file": "DIR/regular_cdb_input2.cpp"
 },
 {
   "directory": "DIR",
   "command": "clang -E DIR/regular_cdb_input.cpp -IInputs",
   "file": "DIR/regular_cdb_input.cpp"
+},
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/regular_cdb_input.cpp -IInputs -o adena.o",
+  "file": "DIR/regular_cdb_input.cpp"
 }
 ]

clang/test/ClangScanDeps/error.cpp

 // CHECK: Error while scanning dependencies
 // CHECK-NEXT: error: no such file or directory:
 // CHECK-NEXT: error: no input files
 // CHECK-NEXT: error:
 // CHECK-NEXT: Error while scanning dependencies
 // CHECK-NEXT: fatal error: 'missing.h' file not found
 // CHECK-NEXT: "missing.h"
 // CHECK-NEXT: ^
+// CHECK-NEXT: Error while scanning dependencies
+// CHECK-NEXT: fatal error: 'missing.h' file not found
+// CHECK-NEXT: "missing.h"
+// CHECK-NEXT: ^
 // CHECK-NEXT: EOF

clang/test/ClangScanDeps/regular_cdb.cpp

 // RUN: rm -rf %t.dir
 // RUN: rm -rf %t.cdb
 // RUN: mkdir -p %t.dir
 // RUN: cp %s %t.dir/regular_cdb_input.cpp
 // RUN: cp %s %t.dir/regular_cdb_input2.cpp
 // RUN: mkdir %t.dir/Inputs
 // RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
 // RUN: cp %S/Inputs/header2.h %t.dir/Inputs/header2.h
 // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb
 //
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources | \
-// RUN:   FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
+// RUN:   FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO,CHECK3 %s
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \
-// RUN:   FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
+// RUN:   FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO,CHECK3 %s
 // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources \
-// RUN:   -skip-excluded-pp-ranges=0 | FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
+// RUN:   -skip-excluded-pp-ranges=0 | FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO,CHECK3 %s
 //
 // Make sure we didn't produce any dependency files!
 // RUN: not cat %t.dir/regular_cdb.d
 // RUN: not cat %t.dir/regular_cdb2.d
 //
 // The output order is non-deterministic when using more than one thread,
 // so check the output using two runs. Note that the 'NOT' check is not used
 // as it might fail if the results for `regular_cdb_input.cpp` are reported before

 
 #include "header.h"
 
 // CHECK1: regular_cdb_input2.cpp
 // CHECK1-NEXT: regular_cdb_input2.cpp
 // CHECK1-NEXT: Inputs{{/|\\}}header.h
 // CHECK1-NEXT: Inputs{{/|\\}}header2.h
 
+// CHECK3: regular_cdb_input.o
 // CHECK2: regular_cdb_input.cpp
 // CHECK2-NEXT: Inputs{{/|\\}}header.h
 // CHECK2NO-NOT: header2
+
+// CHECK3: adena.o

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 
 /// \returns object-file path derived from source-file path.
 static std::string getObjFilePath(StringRef SrcFile) {
   SmallString<128> ObjFileName(SrcFile);
   llvm::sys::path::replace_extension(ObjFileName, "o");
   return ObjFileName.str();
 }
 
+class SingleCommandCompilationDatabase : public tooling::CompilationDatabase {
+public:
+  SingleCommandCompilationDatabase(tooling::CompileCommand Cmd)
+      : Command(std::move(Cmd)) {}
+
+  virtual std::vector<tooling::CompileCommand>
+  getCompileCommands(StringRef FilePath) const {
+    return {Command};
+  }
+
+  virtual std::vector<tooling::CompileCommand> getAllCompileCommands() const {
+    return {Command};
+  }
+
+private:
+  tooling::CompileCommand Command;
+};
+
 /// Takes the result of a dependency scan and prints error / dependency files
 /// based on the result.
 ///
 /// \returns True on error.
 static bool handleDependencyToolResult(const std::string &Input,
                                        llvm::Expected<std::string> &MaybeFile,
                                        SharedStream &OS, SharedStream &Errs) {
   if (!MaybeFile) {
std::unique_ptr<tooling::JSONCompilationDatabase> Compilations =
           tooling::JSONCommandLineSyntax::AutoDetect);
   if (!Compilations) {
     llvm::errs() << "error: " << ErrorMessage << "\n";
     return 1;
   }
 
   llvm::cl::PrintOptionValues();
 
-  // By default the tool runs on all inputs in the CDB.
-  std::vector<std::pair<std::string, std::string>> Inputs;
-  for (const auto &Command : Compilations->getAllCompileCommands())
-    Inputs.emplace_back(Command.Filename, Command.Directory);
-
   // The command options are rewritten to run Clang in preprocessor only mode.
   auto AdjustingCompilations =
       std::make_unique<tooling::ArgumentsAdjustingCompilations>(
           std::move(Compilations));
   AdjustingCompilations->appendArgumentsAdjuster(
       [](const tooling::CommandLineArguments &Args, StringRef FileName) {
         std::string LastO = "";
         bool HasMT = false;

 #if LLVM_ENABLE_THREADS
   unsigned NumWorkers =
       NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
 #else
   unsigned NumWorkers = 1;
 #endif
   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
   for (unsigned I = 0; I < NumWorkers; ++I)
-    WorkerTools.push_back(std::make_unique<DependencyScanningTool>(
-        Service, *AdjustingCompilations));
+    WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
+
+  std::vector<SingleCommandCompilationDatabase> Inputs;
+  for (tooling::CompileCommand Cmd :
+       AdjustingCompilations->getAllCompileCommands())
+    Inputs.emplace_back(Cmd);
 
   std::vector<std::thread> WorkerThreads;
   std::atomic<bool> HadErrors(false);
   std::mutex Lock;
   size_t Index = 0;
 
   if (Verbose) {
     llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
                  << " files using " << NumWorkers << " workers\n";
   }
   for (unsigned I = 0; I < NumWorkers; ++I) {
     auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools,
                    &DependencyOS, &Errs]() {
       while (true) {
-        std::string Input;
-        StringRef CWD;
+        const SingleCommandCompilationDatabase *Input;
+        std::string Filename;
+        std::string CWD;
         // Take the next input.
         {
           std::unique_lock<std::mutex> LockGuard(Lock);
           if (Index >= Inputs.size())
             return;
-          const auto &Compilation = Inputs[Index++];
-          Input = Compilation.first;
-          CWD = Compilation.second;
+          Input = &Inputs[Index++];
+          tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0];
+          Filename = std::move(Cmd.Filename);
+          CWD = std::move(Cmd.Directory);
         }
         // Run the tool on it.
-        auto MaybeFile = WorkerTools[I]->getDependencyFile(Input, CWD);
-        if (handleDependencyToolResult(Input, MaybeFile, DependencyOS, Errs))
+        auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD);
+        if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs))
           HadErrors = true;
       }
     };
 #if LLVM_ENABLE_THREADS
     WorkerThreads.emplace_back(std::move(Worker));
 #else
     // Run the worker without spawning a thread when threads are disabled.
     Worker();
 #endif
   }
   for (auto &W : WorkerThreads)
     W.join();
 
   return HadErrors;
 }