[clang][deps] Stop using `ClangTool` for virtual files

[jansvoboda11]

Summary

This patch changes how the dependency scanner creates the fake input file when scanning dependencies of a single module (introduced in D109485). The scanner now has its own InMemoryFilesystem which sits under the minimizing FS (when that's requested). This makes it possible to drop the duplicate work in DependencyScanningActions::runInvocation that sets up the main file ID. Besides that, this patch makes it possible to land D108979, where we drop ClangTool entirely.

Depends on D109485.

[jansvoboda11]

Updating diff to re-trigger CI...

[dexonsmith]

LGTM, with one nit.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

98

Nit: I suggest "InMemoryFS", which sounds a bit more clear to me.

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

llvm::Error computeDependencies(const std::string &Input,
                                   DependencyConsumer &Consumer,
                                   llvm::Optional<StringRef> ModuleName = None);
 
 private:
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
   std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
 
+  /// The physical filesystem overlaid by `MemoryFS`.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
+  /// The in-memory filesystem laid on top the physical filesystem in `RealFS`.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> MemoryFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists accross multiple compiler
   /// invocations.
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   /// The file manager that is reused accross multiple invocations by this
   /// worker. If null, the file manager will not be reused.
   llvm::IntrusiveRefCntPtr<FileManager> Files;
   ScanningOutputFormat Format;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

 /// A clang tool that runs the preprocessor in a mode that's optimized for
 /// dependency scanning for the given compiler invocation.
 class DependencyScanningAction : public tooling::ToolAction {
 public:
   DependencyScanningAction(
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
-      ScanningOutputFormat Format, llvm::Optional<StringRef> ModuleName = None,
-      llvm::Optional<llvm::MemoryBufferRef> FakeMemBuffer = None)
+      ScanningOutputFormat Format, llvm::Optional<StringRef> ModuleName = None)
       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
-        ModuleName(ModuleName), FakeMemBuffer(FakeMemBuffer) {}
+        ModuleName(ModuleName) {}
 
   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
                      FileManager *FileMgr,
                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
                      DiagnosticConsumer *DiagConsumer) override {
     // Make a deep copy of the original Clang invocation.
     CompilerInvocation OriginalInvocation(*Invocation);
 
if (DepFS) {
 
       // Pass the skip mappings which should speed up excluded conditional block
       // skipping in the preprocessor.
       if (PPSkipMappings)
         Compiler.getPreprocessorOpts()
             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
     }
 
-    if (ModuleName.hasValue()) {
-      SmallString<128> FullPath(*ModuleName);
-      llvm::sys::fs::make_absolute(WorkingDirectory, FullPath);
-      SourceManager &SrcMgr = Compiler.getSourceManager();
-      FileMgr->getVirtualFile(FullPath.c_str(), FakeMemBuffer->getBufferSize(),
-                              0);
-      FileID MainFileID = SrcMgr.createFileID(*FakeMemBuffer);
-      SrcMgr.setMainFileID(MainFileID);
-    }
-
     // Create the dependency collector that will collect the produced
     // dependencies.
     //
     // This also moves the existing dependency output options from the
     // invocation to the collector. The options in the invocation are reset,
     // which ensures that the compiler won't create new dependency collectors,
     // and thus won't write out the extra '.d' files to disk.
     auto Opts = std::make_unique<DependencyOutputOptions>();

 
 private:
   StringRef WorkingDirectory;
   DependencyConsumer &Consumer;
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
   ScanningOutputFormat Format;
   llvm::Optional<StringRef> ModuleName;
-  llvm::Optional<llvm::MemoryBufferRef> FakeMemBuffer;
 };
 
 } // end anonymous namespace
 
 DependencyScanningWorker::DependencyScanningWorker(
     DependencyScanningService &Service)
     : Format(Service.getFormat()) {
   DiagOpts = new DiagnosticOptions();
 
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
   PCHContainerOps->registerReader(
       std::make_unique<ObjectFilePCHContainerReader>());
   // We don't need to write object files, but the current PCH implementation
   // requires the writer to be registered as well.
   PCHContainerOps->registerWriter(
       std::make_unique<ObjectFilePCHContainerWriter>());
 
-  RealFS = llvm::vfs::createPhysicalFileSystem();
+  auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
+      llvm::vfs::createPhysicalFileSystem());
+  MemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  OverlayFS->pushOverlay(MemoryFS);
+  RealFS = OverlayFS;
+
   if (Service.canSkipExcludedPPRanges())
     PPSkipMappings =
         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
     DepFS = new DependencyScanningWorkerFilesystem(
         Service.getSharedCache(), RealFS, PPSkipMappings.get());
   if (Service.canReuseFileManager())
     Files = new FileManager(FileSystemOptions(), RealFS);
return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
                                              llvm::inconvertibleErrorCode());
 }
 
 llvm::Error DependencyScanningWorker::computeDependencies(
     const std::string &Input, StringRef WorkingDirectory,
     const CompilationDatabase &CDB, DependencyConsumer &Consumer,
     llvm::Optional<StringRef> ModuleName) {
   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
-  std::unique_ptr<llvm::MemoryBuffer> FakeMemBuffer =
-      ModuleName.hasValue() ? llvm::MemoryBuffer::getMemBuffer(" ") : nullptr;
   return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) {
     /// Create the tool that uses the underlying file system to ensure that any
     /// file system requests that are made by the driver do not go through the
     /// dependency scanning filesystem.
-    SmallString<128> FullPath;
     tooling::ClangTool Tool(CDB,
                             ModuleName.hasValue() ? ModuleName->str() : Input,
                             PCHContainerOps, RealFS, Files);
     Tool.clearArgumentsAdjusters();
     Tool.setRestoreWorkingDir(false);
     Tool.setPrintErrorMessage(false);
     Tool.setDiagnosticConsumer(&DC);
-    DependencyScanningAction Action(
-        WorkingDirectory, Consumer, DepFS, PPSkipMappings.get(), Format,
-        ModuleName,
-        FakeMemBuffer
-            ? llvm::Optional<llvm::MemoryBufferRef>(*FakeMemBuffer.get())
-            : None);
+    DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
+                                    PPSkipMappings.get(), Format, ModuleName);
 
     if (ModuleName.hasValue()) {
-      Tool.mapVirtualFile(*ModuleName, FakeMemBuffer->getBuffer());
-      FullPath = *ModuleName;
-      llvm::sys::fs::make_absolute(WorkingDirectory, FullPath);
+      MemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(" "));
       Tool.appendArgumentsAdjuster(
           [&](const tooling::CommandLineArguments &Args, StringRef FileName) {
             tooling::CommandLineArguments AdjustedArgs(Args);
-            AdjustedArgs.push_back(std::string(FullPath));
+            AdjustedArgs.emplace_back(*ModuleName);
             return AdjustedArgs;
           });
     }
 
     return !Tool.run(&Action);
   });
 }