WIP: clang-scan-deps: Skip the disk when scanning modules dependencies

[dexonsmith]

Summary

Modules dependendency scanning builds lightweight modules (based on
minimized preprocess-only sources) in order to construct the module
dependency graph. Add -intercept-module-outputs (on-by-default), which
keeps these lightweight modules in memory instead of writing to and
reading from disk, leveraging llvm::vfs::OutputManager.

This patch uses a separate cache per worker, but these would ideally be
kept in a shared result cache (could be an in-memory CAS).

This is a WIP because it's based on the OutputManager RFC (see patches https://reviews.llvm.org/D95501 / https://reviews.llvm.org/D95502) but I'm posting this as well since it's a simple example usage.

I haven't benchmarked (yet); it's motivated by profiling from a couple of years ago that showed some slowdowns here.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

 };
 
 /// The dependency scanning service contains the shared state that is used by
 /// the invidual dependency scanning workers.
 class DependencyScanningService {
 public:
   DependencyScanningService(ScanningMode Mode, ScanningOutputFormat Format,
                             bool ReuseFileManager = true,
-                            bool SkipExcludedPPRanges = true);
+                            bool SkipExcludedPPRanges = true,
+                            bool InterceptModuleOutputs = true);
 
   ScanningMode getMode() const { return Mode; }
 
   ScanningOutputFormat getFormat() const { return Format; }
 
   bool canReuseFileManager() const { return ReuseFileManager; }
 
+  bool interceptModuleOutputs() const { return InterceptModuleOutputs; }
+
   bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
 
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
 
 private:
   const ScanningMode Mode;
   const ScanningOutputFormat Format;
   const bool ReuseFileManager;
   /// Set to true to use the preprocessor optimization that skips excluded PP
   /// ranges by bumping the buffer pointer in the lexer instead of lexing the
   /// tokens in the range until reaching the corresponding directive.
   const bool SkipExcludedPPRanges;
+  /// Whether to intercept modules in-memory to avoid disk traffic.
+  const bool InterceptModuleOutputs;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

 #include "clang/Basic/LLVM.h"
 #include "clang/Frontend/PCHContainerOperations.h"
 #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/OutputManager.h"
 #include <string>
 
 namespace clang {
 
 class DependencyOutputOptions;
 
 namespace tooling {
 namespace dependencies {
llvm::Error computeDependencies(const std::string &Input,
                                   const CompilationDatabase &CDB,
                                   DependencyConsumer &Consumer);
 
 private:
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
   std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
 
+  /// Output manager to use.
+  std::shared_ptr<llvm::vfs::OutputManager> Outputs;
+
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists accross multiple compiler
   /// invocations.
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   /// The file manager that is reused accross multiple invocations by this
   /// worker. If null, the file manager will not be reused.
   llvm::IntrusiveRefCntPtr<FileManager> Files;
   ScanningOutputFormat Format;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H

clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp

 //===- DependencyScanningService.cpp - clang-scan-deps service ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
 DependencyScanningService::DependencyScanningService(
     ScanningMode Mode, ScanningOutputFormat Format, bool ReuseFileManager,
-    bool SkipExcludedPPRanges)
+    bool SkipExcludedPPRanges, bool InterceptModuleOutputs)
     : Mode(Mode), Format(Format), ReuseFileManager(ReuseFileManager),
-      SkipExcludedPPRanges(SkipExcludedPPRanges) {}
+      SkipExcludedPPRanges(SkipExcludedPPRanges),
+      InterceptModuleOutputs(InterceptModuleOutputs) {}

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

 };
 
 /// A clang tool that runs the preprocessor in a mode that's optimized for
 /// dependency scanning for the given compiler invocation.
 class DependencyScanningAction : public tooling::ToolAction {
 public:
   DependencyScanningAction(
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
+      std::shared_ptr<llvm::vfs::OutputManager> Outputs,
       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
       ScanningOutputFormat Format)
       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
-        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings),
-        Format(Format) {}
+        Outputs(std::move(Outputs)), DepFS(std::move(DepFS)),
+        PPSkipMappings(PPSkipMappings), Format(Format) {}
 
   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
                      FileManager *FileMgr,
                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
                      DiagnosticConsumer *DiagConsumer) override {
     // Create a compiler instance to handle the actual work.
     CompilerInstance Compiler(std::move(PCHContainerOps));
     Compiler.setInvocation(std::move(Invocation));
 
+    // Configure the output manager.
+    if (Outputs) {
+      Compiler.setOutputManager(std::move(Outputs));
+
+      // FIXME: Is this the best way to set the flag?
+      Compiler.getFrontendOpts().BuildingImplicitModuleUsesLock = false;
+    }
+
     // Don't print 'X warnings and Y errors generated'.
     Compiler.getDiagnosticOpts().ShowCarets = false;
     // Create the compiler's actual diagnostics engine.
     Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
     if (!Compiler.hasDiagnostics())
       return false;
 
     // Use the dependency scanning optimized file system if we can.
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
     if (!DepFS)
       FileMgr->clearStatCache();
     return Result;
   }
 
 private:
   StringRef WorkingDirectory;
   DependencyConsumer &Consumer;
+  std::shared_ptr<llvm::vfs::OutputManager> Outputs;
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
   ScanningOutputFormat Format;
 };
 
 } // end anonymous namespace
 
 DependencyScanningWorker::DependencyScanningWorker(
     DependencyScanningService &Service)
     : Format(Service.getFormat()) {
   DiagOpts = new DiagnosticOptions();
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
   RealFS = llvm::vfs::createPhysicalFileSystem();
+
+  if (Service.interceptModuleOutputs()) {
+    using namespace llvm;
+    using namespace llvm::vfs;
+
+    // Create an in-memory filesystem for storing modules and overlay it on top
+    // of RealFS.
+    auto InMemoryFS = makeIntrusiveRefCnt<InMemoryFileSystem>();
+    {
+      auto OverlayFS =
+        makeIntrusiveRefCnt<OverlayFileSystem>(createPhysicalFileSystem());
+      OverlayFS->pushOverlay(InMemoryFS);
+      RealFS = std::move(OverlayFS);
+    }
+
+    // Capture outputs with a ".pcm" extension and save them in ModulesCache.
+    // Don't reject attempts to overwrite (it'll still reject if the content
+    // doesn't match).
+    //
+    // FIXME: The workers could share a single cache using an in-memory CAS.
+    auto InMemoryOutputs =
+        makeIntrusiveRefCnt<InMemoryOutputBackend>(std::move(InMemoryFS));
+    InMemoryOutputs->setShouldRejectExistingFiles(false);
+    Outputs = std::make_shared<OutputManager>(makeFilteringOutputBackend(
+        std::move(InMemoryOutputs), [](StringRef OutputPath, OutputConfig) {
+          return llvm::sys::path::extension(OutputPath) == ".pcm";
+        }));
+  }
+
   if (Service.canSkipExcludedPPRanges())
     PPSkipMappings =
         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
     DepFS = new DependencyScanningWorkerFilesystem(
         Service.getSharedCache(), RealFS, PPSkipMappings.get());
   if (Service.canReuseFileManager())
     Files = new FileManager(FileSystemOptions(), RealFS);
return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) {
     /// Create the tool that uses the underlying file system to ensure that any
     /// file system requests that are made by the driver do not go through the
     /// dependency scanning filesystem.
     tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS, Files);
     Tool.clearArgumentsAdjusters();
     Tool.setRestoreWorkingDir(false);
     Tool.setPrintErrorMessage(false);
     Tool.setDiagnosticConsumer(&DC);
-    DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
+    DependencyScanningAction Action(WorkingDirectory, Consumer, Outputs, DepFS,
                                     PPSkipMappings.get(), Format);
     return !Tool.run(&Action);
   });
 }

clang/tools/clang-scan-deps/ClangScanDeps.cpp

 llvm::cl::opt<bool> SkipExcludedPPRanges(
     "skip-excluded-pp-ranges",
     llvm::cl::desc(
         "Use the preprocessor optimization that skips excluded conditionals by "
         "bumping the buffer pointer in the lexer instead of lexing the tokens  "
         "until reaching the end directive."),
     llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
 
+llvm::cl::opt<bool> InterceptModuleOutputs(
+    "intercept-module-outputs",
+    llvm::cl::desc(
+        "Keep the module outputs in-memory only, skipping the on-disk cache."),
+    llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+
 llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional,
                             llvm::cl::desc("Use verbose output."),
                             llvm::cl::init(false),
                             llvm::cl::cat(DependencyScannerCategory));
 
 } // end anonymous namespace
 
 /// \returns object-file path derived from source-file path.
int main(int argc, const char **argv) {
   AdjustingCompilations->appendArgumentsAdjuster(
       tooling::getClangStripSerializeDiagnosticAdjuster());
 
   SharedStream Errs(llvm::errs());
   // Print out the dependency results to STDOUT by default.
   SharedStream DependencyOS(llvm::outs());
 
   DependencyScanningService Service(ScanMode, Format, ReuseFileManager,
-                                    SkipExcludedPPRanges);
+                                    SkipExcludedPPRanges,
+                                    InterceptModuleOutputs);
   llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads));
   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
   for (unsigned I = 0; I < Pool.getThreadCount(); ++I)
     WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
 
   std::vector<SingleCommandCompilationDatabase> Inputs;
   for (tooling::CompileCommand Cmd :
        AdjustingCompilations->getAllCompileCommands())