Support `ivfsoverlay` option in Tooling

[vladimir.plyashkun]

Summary

Previously, this argument had no effect, since it didn't proceeded.
For more information, check this review: https://reviews.llvm.org/D41535

[ilya-biryukov]

I'm not sure if it's ok to ignore the shared FileManager here.
@klimek, @bkramer, @alexfh, does tooling library rely on having the same FileManager for all invocations? Is it just a performance optimization or there's more to it?

lib/Tooling/Tooling.cpp

287

Files is a raw pointer, so we're leaking memory here.

[vladimir.plyashkun]

lib/Tooling/Tooling.cpp

287

Agree.
I can try to replace type of this field to llvm::IntrusiveRefCntPtr<T> instead of raw pointer.
But if i understand correctly, this class is available during whole execution, so memory will be freed on exit.
I saw some other usages, for example, createFileManager method in the CompilerInstance also just reassign value to raw pointer.
https://clang.llvm.org/doxygen/classclang_1_1CompilerInstance.html#abeb2bbf46a8de987c227125a84935802

[ilya-biryukov]

lib/Tooling/Tooling.cpp

287

Using IntrusiveRefCntPtr<FileManager> locally should do the trick, the clients can take ownership if they want and FileManager will be properly freed if they don't do that.

CompilerInstance::createFileManager stores IntrusiveRefCntPtr as a field before returning a raw pointer, so it seems to properly manage memory there.

[ilya-biryukov]

With D41947 in place, do we still need this change? Or can it be "abandoned" now?

lib/Frontend/CompilerInvocation.cpp

createVFSFromCompilerInvocation(const CompilerInvocation &CI,
   for (const std::string &File : CI.getHeaderSearchOpts().VFSOverlayFiles) {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
         BaseFS->getBufferForFile(File);
     if (!Buffer) {
       Diags.Report(diag::err_missing_vfs_overlay_file) << File;
       return IntrusiveRefCntPtr<vfs::FileSystem>();
     }
 
-    IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getVFSFromYAML(
-        std::move(Buffer.get()), /*DiagHandler*/ nullptr, File);
+    IntrusiveRefCntPtr<vfs::FileSystem> FS =
+        vfs::getVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr,
+                            File, /*DiagContext*/ nullptr, BaseFS);
     if (!FS.get()) {
       Diags.Report(diag::err_invalid_vfs_overlay) << File;
       return IntrusiveRefCntPtr<vfs::FileSystem>();
     }
     Overlay->pushOverlay(FS);
   }
   return Overlay;
 }
 } // end namespace clang

lib/Tooling/Tooling.cpp

bool ToolInvocation::run() {
   // FIXME: remove this when all users have migrated!
   for (const auto &It : MappedFileContents) {
     // Inject the code as the given file name into the preprocessor options.
     std::unique_ptr<llvm::MemoryBuffer> Input =
         llvm::MemoryBuffer::getMemBuffer(It.getValue());
     Invocation->getPreprocessorOpts().addRemappedFile(It.getKey(),
                                                       Input.release());
   }
+  IntrusiveRefCntPtr<vfs::FileSystem> VirtualFileSystem =
+      createVFSFromCompilerInvocation(*Invocation, Diagnostics,
+                                      Files->getVirtualFileSystem());
+  if (Files->getVirtualFileSystem() != VirtualFileSystem) {
+    Files = new FileManager(Invocation->getFileSystemOpts(), VirtualFileSystem);
+  }
   return runInvocation(BinaryName, Compilation.get(), std::move(Invocation),
                        std::move(PCHContainerOps));
 }
 
 bool ToolInvocation::runInvocation(
     const char *BinaryName, clang::driver::Compilation *Compilation,
     std::shared_ptr<clang::CompilerInvocation> Invocation,
     std::shared_ptr<PCHContainerOperations> PCHContainerOps) {

unittests/Tooling/ToolingTest.cpp

 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "gtest/gtest.h"
 #include <algorithm>
 #include <string>
 
 namespace clang {
 namespace tooling {
 
 namespace {
clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction,
                                             Files.get());
   InMemoryFileSystem->addFile(
       "test.cpp", 0, llvm::MemoryBuffer::getMemBuffer("#include <abc>\n"));
   InMemoryFileSystem->addFile("def/abc", 0,
                               llvm::MemoryBuffer::getMemBuffer("\n"));
   EXPECT_TRUE(Invocation.run());
 }
 
+TEST(ToolInvocation, TestVfsOverlay) {
+  llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem(
+      new vfs::OverlayFileSystem(vfs::getRealFileSystem()));
+  llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem(
+      new vfs::InMemoryFileSystem);
+  OverlayFileSystem->pushOverlay(InMemoryFileSystem);
+  llvm::IntrusiveRefCntPtr<FileManager> Files(
+      new FileManager(FileSystemOptions(), OverlayFileSystem));
+  std::vector<std::string> Args;
+  Args.push_back("tool-executable");
+  Args.push_back("-ivfsoverlay");
+  Args.push_back("overlay.yaml");
+  Args.push_back("-fsyntax-only");
+  Args.push_back("a.cpp");
+  clang::tooling::ToolInvocation Invocation(Args, new SyntaxOnlyAction,
+                                            Files.get());
+  InMemoryFileSystem->addFile(
+      "a.cpp", 0,
+      llvm::MemoryBuffer::getMemBuffer("#include <unknown_header>\n"));
+  std::string OverlayContent = llvm::formatv(
+      "{{'version': 0,'roots':[{{'name':'{0}','type':'directory',"
+    "'contents':[{{'name':'a.cpp','type':'file',"
+    "'external-contents':'b.cpp'}]}]}",
+      InMemoryFileSystem.get()->getCurrentWorkingDirectory().get());
+  InMemoryFileSystem->addFile("overlay.yaml", 0,
+                              llvm::MemoryBuffer::getMemBuffer(OverlayContent));
+  InMemoryFileSystem->addFile(
+      "b.cpp", 0, llvm::MemoryBuffer::getMemBuffer("int main() {}"));
+  EXPECT_TRUE(Invocation.run());
+}
+
 TEST(ToolInvocation, TestVirtualModulesCompilation) {
   // FIXME: Currently, this only tests that we don't exit with an error if a
   // mapped module.map is found on the include path. In the future, expand this
   // test to run a full modules enabled compilation, so we make sure we can
   // rerun modules compilations with a virtual file system.
   llvm::IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem(
       new vfs::OverlayFileSystem(vfs::getRealFileSystem()));
   llvm::IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem(