[clang][deps] Abolish FileManager sharing

[jansvoboda11]

Summary

This patch removes the ability of a dependency scanning worker to share a FileManager instance between individual scans. It's not sound and doesn't provide performance benefits (due to the underlying caching VFS).

[benlangmuir]

Thanks for cleaning this up!

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

enum class ScanningOutputFormat {
   /// intermodule dependency information.
   Make,
 
   /// This outputs the full module dependency graph suitable for use for
   /// explicitly building modules.
   Full,
 };
 
-/// The dependency scanning service contains the shared state that is used by
-/// the invidual dependency scanning workers.
+/// The dependency scanning service contains shared configuration and state that
+/// is used by the individual dependency scanning workers.
 class DependencyScanningService {
 public:
   DependencyScanningService(ScanningMode Mode, ScanningOutputFormat Format,
-                            bool ReuseFileManager = true,
                             bool OptimizeArgs = false,
                             bool EagerLoadModules = false);
 
   ScanningMode getMode() const { return Mode; }
 
   ScanningOutputFormat getFormat() const { return Format; }
 
-  bool canReuseFileManager() const { return ReuseFileManager; }
-
   bool canOptimizeArgs() const { return OptimizeArgs; }
 
   bool shouldEagerLoadModules() const { return EagerLoadModules; }
 
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
 
 private:
   const ScanningMode Mode;
   const ScanningOutputFormat Format;
-  const bool ReuseFileManager;
   /// Whether to optimize the modules' command-line arguments.
   const bool OptimizeArgs;
   /// Whether to set up command-lines to load PCM files eagerly.
   const bool EagerLoadModules;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

private:
   /// The physical filesystem overlaid by `InMemoryFS`.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The in-memory filesystem laid on top the physical filesystem in `RealFS`.
   llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists across multiple compiler
   /// invocations.
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
-  /// The file manager that is reused across multiple invocations by this
-  /// worker. If null, the file manager will not be reused.
-  llvm::IntrusiveRefCntPtr<FileManager> Files;
   ScanningOutputFormat Format;
   /// Whether to optimize the modules' command-line arguments.
   bool OptimizeArgs;
   /// Whether to set up command-lines to load PCM files eagerly.
   bool EagerLoadModules;
 };
 
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
 
 #endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H

clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp

 //===- DependencyScanningService.cpp - clang-scan-deps service ------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "llvm/Support/TargetSelect.h"
 
 using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
 DependencyScanningService::DependencyScanningService(
-    ScanningMode Mode, ScanningOutputFormat Format, bool ReuseFileManager,
-    bool OptimizeArgs, bool EagerLoadModules)
-    : Mode(Mode), Format(Format), ReuseFileManager(ReuseFileManager),
-      OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules) {
+    ScanningMode Mode, ScanningOutputFormat Format, bool OptimizeArgs,
+    bool EagerLoadModules)
+    : Mode(Mode), Format(Format), OptimizeArgs(OptimizeArgs),
+      EagerLoadModules(EagerLoadModules) {
   // Initialize targets for object file support.
   llvm::InitializeAllTargets();
   llvm::InitializeAllTargetMCs();
   llvm::InitializeAllAsmPrinters();
   llvm::InitializeAllAsmParsers();
 }

clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp

bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
     std::unique_ptr<FrontendAction> Action;
 
     if (ModuleName)
       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
     else
       Action = std::make_unique<ReadPCHAndPreprocessAction>();
 
     const bool Result = ScanInstance.ExecuteAction(*Action);
-    if (!DepFS)
-      FileMgr->clearStatCache();
 
     if (Result)
       setLastCC1Arguments(std::move(OriginalInvocation));
 
     return Result;
   }
 
   bool hasScanned() const { return Scanned; }
auto OverlayFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(std::move(FS));
   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
   OverlayFS->pushOverlay(InMemoryFS);
   RealFS = OverlayFS;
 
   if (Service.getMode() == ScanningMode::DependencyDirectivesScan)
     DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
                                                    RealFS);
-  if (Service.canReuseFileManager())
-    Files = new FileManager(FileSystemOptions(), RealFS);
 }
 
 llvm::Error DependencyScanningWorker::computeDependencies(
     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
   std::vector<const char *> CLI;
   for (const std::string &Arg : CommandLine)
     CLI.push_back(Arg.c_str());

 }
 
 bool DependencyScanningWorker::computeDependencies(
     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
     DependencyConsumer &Consumer, DiagnosticConsumer &DC,
     llvm::Optional<StringRef> ModuleName) {
   // Reset what might have been modified in the previous worker invocation.
   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
-  if (Files)
-    Files->setVirtualFileSystem(RealFS);
 
-  llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
-      Files ? Files : new FileManager(FileSystemOptions(), RealFS);
+  auto FileMgr =
+      llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions(), RealFS);
 
   Optional<std::vector<std::string>> ModifiedCommandLine;
   if (ModuleName) {
     ModifiedCommandLine = CommandLine;
     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
     ModifiedCommandLine->emplace_back(*ModuleName);
   }
 
   const std::vector<std::string> &FinalCommandLine =
       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
 
   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
   llvm::transform(CommandLine, FinalCCommandLine.begin(),
                   [](const std::string &Str) { return Str.c_str(); });
 
   auto DiagOpts = CreateAndPopulateDiagOpts(FinalCCommandLine);
   sanitizeDiagOpts(*DiagOpts);
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
       CompilerInstance::createDiagnostics(DiagOpts.release(), &DC,
                                           /*ShouldOwnClient=*/false);
 
   // Although `Diagnostics` are used only for command-line parsing, the
   // custom `DiagConsumer` might expect a `SourceManager` to be present.
-  SourceManager SrcMgr(*Diags, *CurrentFiles);
+  SourceManager SrcMgr(*Diags, *FileMgr);
   Diags->setSourceManager(&SrcMgr);
   // DisableFree is modified by Tooling for running
   // in-process; preserve the original value, which is
   // always true for a driver invocation.
   bool DisableFree = true;
   DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS, Format,
                                   OptimizeArgs, EagerLoadModules, DisableFree,
                                   ModuleName);
   bool Success = forEachDriverJob(
-      FinalCommandLine, *Diags, *CurrentFiles, [&](const driver::Command &Cmd) {
+      FinalCommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
         if (StringRef(Cmd.getCreator().getName()) != "clang") {
           // Non-clang command. Just pass through to the dependency
           // consumer.
           Consumer.handleBuildCommand(
               {Cmd.getExecutable(),
                {Cmd.getArguments().begin(), Cmd.getArguments().end()}});
           return true;
         }
 
         std::vector<std::string> Argv;
         Argv.push_back(Cmd.getExecutable());
         Argv.insert(Argv.end(), Cmd.getArguments().begin(),
                     Cmd.getArguments().end());
 
         // Create an invocation that uses the underlying file
         // system to ensure that any file system requests that
         // are made by the driver do not go through the
         // dependency scanning filesystem.
-        ToolInvocation Invocation(std::move(Argv), &Action, &*CurrentFiles,
+        ToolInvocation Invocation(std::move(Argv), &Action, &*FileMgr,
                                   PCHContainerOps);
         Invocation.setDiagnosticConsumer(Diags->getClient());
         Invocation.setDiagnosticOptions(&Diags->getDiagnosticOptions());
         if (!Invocation.run())
           return false;
 
         std::vector<std::string> Args = Action.takeLastCC1Arguments();
         Consumer.handleBuildCommand({Cmd.getExecutable(), std::move(Args)});
         return true;
       });
 
   if (Success && !Action.hasScanned())
     Diags->Report(diag::err_fe_expected_compiler_job)
         << llvm::join(FinalCommandLine, " ");
   return Success && Action.hasScanned();
 }

clang/test/ClangScanDeps/modulemap-via-vfs.m

 // RUN: rm -rf %t.dir
 // RUN: split-file %s %t.dir
 // RUN: sed -e "s|DIR|%/t.dir|g" %t.dir/build/compile-commands.json.in > %t.dir/build/compile-commands.json
 // RUN: sed -e "s|DIR|%/t.dir|g" %t.dir/build/vfs.yaml.in > %t.dir/build/vfs.yaml
 // RUN: clang-scan-deps -compilation-database %t.dir/build/compile-commands.json \
-// RUN:   -reuse-filemanager=0 -j 1 -format experimental-full \
-// RUN:   -mode preprocess-dependency-directives > %t.db
+// RUN:   -j 1 -format experimental-full -mode preprocess-dependency-directives > %t.db
 // RUN: %deps-to-rsp %t.db --module-name=A > %t.A.cc1.rsp
 // RUN: cat %t.A.cc1.rsp | sed 's:\\\\\?:/:g' | FileCheck %s
 
 // CHECK-NOT: build/module.modulemap
 // CHECK: A/module.modulemap
 // CHECK-NOT: build/module.modulemap
 
 //--- build/compile-commands.json.in

clang/test/ClangScanDeps/subframework_header_dir_symlink.m

 // REQUIRES: shell
 // RUN: rm -rf %t.dir
 // RUN: rm -rf %t.cdb
 // RUN: mkdir -p %t.dir
 // RUN: cp %s %t.dir/subframework_header_dir_symlink_input.m
 // RUN: cp %s %t.dir/subframework_header_dir_symlink_input2.m
 // RUN: mkdir %t.dir/Inputs
 // RUN: cp -R %S/Inputs/frameworks %t.dir/Inputs/frameworks
 // RUN: ln -s %t.dir/Inputs/frameworks %t.dir/Inputs/frameworks_symlink
 // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/subframework_header_dir_symlink_cdb.json > %t.cdb
-// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=0 | \
-// RUN:   FileCheck %s
-// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=1 | \
-// RUN:   FileCheck %s
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 |  FileCheck %s
 
 #ifndef EMPTY
 #include "Framework/Framework.h"
 #endif
 
 // CHECK: subframework_header_dir_symlink_input.o
 // CHECK-NEXT: subframework_header_dir_symlink_input.m
 // CHECK: subframework_header_dir_symlink_input2.o
 // CHECK-NEXT: subframework_header_dir_symlink_input2.m
 // CHECK-NEXT: Inputs{{/|\\}}frameworks_symlink{{/|\\}}Framework.framework{{/|\\}}Headers{{/|\\}}Framework.h

clang/test/ClangScanDeps/symlink.cpp

 // REQUIRES: shell
 // RUN: rm -rf %t.dir
 // RUN: rm -rf %t.cdb
 // RUN: mkdir -p %t.dir
 // RUN: cp %s %t.dir/symlink_input.cpp
 // RUN: cp %s %t.dir/symlink_input2.cpp
 // RUN: mkdir %t.dir/Inputs
 // RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
 // RUN: ln -s %t.dir/Inputs/header.h %t.dir/Inputs/symlink.h
 // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/symlink_cdb.json > %t.cdb
-// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=0 | FileCheck %s
-// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=1 | FileCheck %s
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 | FileCheck %s
 
 #include "symlink.h"
 #include "header.h"
 
 // CHECK: symlink_input.cpp
 // CHECK-NEXT: Inputs{{/|\\}}symlink.h
 // CHECK-NEXT: Inputs{{/|\\}}header.h
 
 // CHECK: symlink_input2.cpp
 // CHECK-NEXT: Inputs{{/|\\}}symlink.h
 // CHECK-NEXT: Inputs{{/|\\}}header.h

clang/tools/clang-scan-deps/ClangScanDeps.cpp

NumThreads("j", llvm::cl::Optional,
                               "all concurrent threads)"),
                llvm::cl::init(0), llvm::cl::cat(DependencyScannerCategory));
 
 llvm::cl::opt<std::string>
     CompilationDB("compilation-database",
                   llvm::cl::desc("Compilation database"), llvm::cl::Required,
                   llvm::cl::cat(DependencyScannerCategory));
 
-llvm::cl::opt<bool> ReuseFileManager(
-    "reuse-filemanager",
-    llvm::cl::desc("Reuse the file manager and its cache between invocations."),
-    llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
-
 llvm::cl::opt<std::string> ModuleName(
     "module-name", llvm::cl::Optional,
     llvm::cl::desc("the module of which the dependencies are to be computed"),
     llvm::cl::cat(DependencyScannerCategory));
 
 llvm::cl::list<std::string> ModuleDepTargets(
     "dependency-target",
     llvm::cl::desc("The names of dependency targets for the dependency file"),
AdjustingCompilations->appendArgumentsAdjuster(
         AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end());
         return AdjustedArgs;
       });
 
   SharedStream Errs(llvm::errs());
   // Print out the dependency results to STDOUT by default.
   SharedStream DependencyOS(llvm::outs());
 
-  DependencyScanningService Service(ScanMode, Format, ReuseFileManager,
-                                    OptimizeArgs, EagerLoadModules);
+  DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
+                                    EagerLoadModules);
   llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads));
   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
   for (unsigned I = 0; I < Pool.getThreadCount(); ++I)
     WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
 
   std::vector<tooling::CompileCommand> Inputs =
       AdjustingCompilations->getAllCompileCommands();