[Driver] Add -ivfsoverlay-lib option to load VFS from shared library

[asavonic]

Summary

This patch allows clang to use a VirtualFileSystem represented by a
shared library. This VFS will be used on top of the real file system.

Library specified -ivfsoverlay-lib by must export
`clang::vfs::FileSystem *__clang_create_vfs()' function.

Unlike -ivfsoverlay option, which allows to map one real file to
another, -ivfsoverlay-lib allows to use files from memory, or do any
kind of files preprocessing.

[asavonic]

Add license header.

[asavonic]

Fixed warning about void* to function pointer cast.

[JDevlieghere]

I don't know much about the VFS so I only did a superficial review.

include/clang/Lex/HeaderSearchOptions.h

176

We now have autobrief enabled so you can omit the \brief.

lib/Frontend/CompilerInvocation.cpp

3218

Would it be useful to print the contents of the error?

unittests/Frontend/VfsTestLib.cpp

25

The llvm coding standard says no braces for single line statements.

[asavonic]

Addressed CR comments, added a diagnostic if shared library was not loaded.

include/clang/Basic/DiagnosticFrontendKinds.td

 def err_test_module_file_extension_version : Error<
   "test module file extension '%0' has different version (%1.%2) than expected "
   "(%3.%4)">;
 
 def err_missing_vfs_overlay_file : Error<
   "virtual filesystem overlay file '%0' not found">, DefaultFatal;
 def err_invalid_vfs_overlay : Error<
   "invalid virtual filesystem overlay file '%0'">, DefaultFatal;
+def err_unable_to_load_vfs_overlay : Error<
+  "unable to load virtual filesystem overlay '%0': %1">, DefaultFatal;
 
 def warn_option_invalid_ocl_version : Warning<
   "OpenCL version %0 does not support the option '%1'">, InGroup<Deprecated>;
 }

include/clang/Driver/Options.td

 def iwithprefix : JoinedOrSeparate<["-"], "iwithprefix">, Group<clang_i_Group>, Flags<[CC1Option]>,
   HelpText<"Set directory to SYSTEM include search path with prefix">, MetaVarName<"<dir>">;
 def iwithsysroot : JoinedOrSeparate<["-"], "iwithsysroot">, Group<clang_i_Group>,
   HelpText<"Add directory to SYSTEM include search path, "
            "absolute paths are relative to -isysroot">, MetaVarName<"<directory>">,
   Flags<[CC1Option]>;
 def ivfsoverlay : JoinedOrSeparate<["-"], "ivfsoverlay">, Group<clang_i_Group>, Flags<[CC1Option]>,
   HelpText<"Overlay the virtual filesystem described by file over the real file system">;
+def ivfsoverlay_lib : JoinedOrSeparate<["-"], "ivfsoverlay-lib">, Group<clang_i_Group>, Flags<[CC1Option]>,
+  HelpText<"Load the virtual filesystem from shared library">;
 def imultilib : Separate<["-"], "imultilib">, Group<gfortran_Group>;
 def keep__private__externs : Flag<["-"], "keep_private_externs">;
 def l : JoinedOrSeparate<["-"], "l">, Flags<[LinkerInput, RenderJoined]>,
         Group<Link_Group>;
 def lazy__framework : Separate<["-"], "lazy_framework">, Flags<[LinkerInput]>;
 def lazy__library : Separate<["-"], "lazy_library">, Flags<[LinkerInput]>;
 def mlittle_endian : Flag<["-"], "mlittle-endian">, Flags<[DriverOption]>;
 def EL : Flag<["-"], "EL">, Alias<mlittle_endian>;

include/clang/Lex/HeaderSearchOptions.h

public:
 
   /// The set of macro names that should be ignored for the purposes
   /// of computing the module hash.
   llvm::SmallSetVector<llvm::CachedHashString, 16> ModulesIgnoreMacros;
 
   /// The set of user-provided virtual filesystem overlay files.
   std::vector<std::string> VFSOverlayFiles;
 
+  /// The set of libraries with user-provided virtual filesystem.
+  std::vector<std::string> VFSOverlayLibs;
+
   /// Include the compiler builtin includes.
   unsigned UseBuiltinIncludes : 1;
 
   /// Include the system standard include search directories.
   unsigned UseStandardSystemIncludes : 1;
 
   /// Include the system standard C++ library include search directories.
   unsigned UseStandardCXXIncludes : 1;
public:
   void AddSystemHeaderPrefix(StringRef Prefix, bool IsSystemHeader) {
     SystemHeaderPrefixes.emplace_back(Prefix, IsSystemHeader);
   }
 
   void AddVFSOverlayFile(StringRef Name) {
     VFSOverlayFiles.push_back(Name);
   }
 
+  void AddVFSOverlayLib(StringRef Name) {
+    VFSOverlayLibs.push_back(Name);
+  }
+
   void AddPrebuiltModulePath(StringRef Name) {
     PrebuiltModulePaths.push_back(Name);
   }
 };
 
 } // namespace clang
 
 #endif // LLVM_CLANG_LEX_HEADERSEARCHOPTIONS_H

lib/Frontend/CompilerInvocation.cpp

 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/OptSpecifier.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Option/Option.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
   // Add the path prefixes which are implicitly treated as being system headers.
   for (const auto *A :
        Args.filtered(OPT_system_header_prefix, OPT_no_system_header_prefix))
     Opts.AddSystemHeaderPrefix(
         A->getValue(), A->getOption().matches(OPT_system_header_prefix));
 
   for (const auto *A : Args.filtered(OPT_ivfsoverlay))
     Opts.AddVFSOverlayFile(A->getValue());
+
+  for (const Arg *A : Args.filtered(OPT_ivfsoverlay_lib))
+    Opts.AddVFSOverlayLib(A->getValue());
 }
 
 void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
                                          const llvm::Triple &T,
                                          PreprocessorOptions &PPOpts,
                                          LangStandard::Kind LangStd) {
   // Set some properties which depend solely on the input kind; it would be nice
   // to move these to the language standard, and have the driver resolve the

 }
 
 IntrusiveRefCntPtr<vfs::FileSystem>
 createVFSFromCompilerInvocation(const CompilerInvocation &CI,
                                 DiagnosticsEngine &Diags) {
   return createVFSFromCompilerInvocation(CI, Diags, vfs::getRealFileSystem());
 }
 
+typedef vfs::FileSystem *(*CreateLibraryFileSystem)(void);
+
 IntrusiveRefCntPtr<vfs::FileSystem>
 createVFSFromCompilerInvocation(const CompilerInvocation &CI,
                                 DiagnosticsEngine &Diags,
                                 IntrusiveRefCntPtr<vfs::FileSystem> BaseFS) {
-  if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty())
+  if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty() &&
+      CI.getHeaderSearchOpts().VFSOverlayLibs.empty())
     return BaseFS;
 
   IntrusiveRefCntPtr<vfs::OverlayFileSystem> Overlay(
       new vfs::OverlayFileSystem(BaseFS));
   // earlier vfs files are on the bottom
   for (const auto &File : CI.getHeaderSearchOpts().VFSOverlayFiles) {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
         BaseFS->getBufferForFile(File);
     if (!Buffer) {
       Diags.Report(diag::err_missing_vfs_overlay_file) << File;
       continue;
     }
 
     IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getVFSFromYAML(
         std::move(Buffer.get()), /*DiagHandler*/ nullptr, File);
     if (FS)
       Overlay->pushOverlay(FS);
     else
       Diags.Report(diag::err_invalid_vfs_overlay) << File;
   }
+
+  // Load shared libraries that provide a VFS.
+  for (const std::string &LibFile : CI.getHeaderSearchOpts().VFSOverlayLibs) {
+    std::string Error;
+    auto Lib = llvm::sys::DynamicLibrary::getPermanentLibrary(
+        LibFile.c_str(), &Error);
+    if (!Lib.isValid()) {
+      Diags.Report(diag::err_unable_to_load_vfs_overlay) << LibFile << Error;
+      continue;
+    }
+
+    auto *CreateFS = (CreateLibraryFileSystem) (intptr_t)
+        Lib.getAddressOfSymbol("__clang_create_vfs");
+
+    if (!CreateFS) {
+      Diags.Report(diag::err_unable_to_load_vfs_overlay)
+        << LibFile << "'__clang_create_vfs' function was not found";
+      continue;
+    }
+
+    IntrusiveRefCntPtr<vfs::FileSystem> FS = CreateFS();
+    if (!FS.get()) {
+      Diags.Report(diag::err_invalid_vfs_overlay) << LibFile;
+      continue;
+    }
+
+    Overlay->pushOverlay(FS);
+  }
   return Overlay;
 }
 
 } // namespace clang

unittests/Frontend/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
+list(APPEND LLVM_OPTIONAL_SOURCES VfsTestLib.cpp)
 
 add_clang_unittest(FrontendTests
   ASTUnitTest.cpp
   CompilerInstanceTest.cpp
   FrontendActionTest.cpp
   CodeGenActionTest.cpp
   ParsedSourceLocationTest.cpp
   PCHPreambleTest.cpp
   OutputStreamTest.cpp
   )
 target_link_libraries(FrontendTests
   PRIVATE
   clangAST
   clangBasic
   clangFrontend
   clangLex
   clangSema
   clangCodeGen
   clangFrontendTool
   )
+
+add_library(VfsTestLib SHARED VfsTestLib.cpp)
+target_link_libraries(VfsTestLib clangBasic)
+set_target_properties(VfsTestLib
+  PROPERTIES PREFIX ""
+  SUFFIX ${LTDL_SHLIB_EXT}
+  )
+set(outdir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR})
+set_output_directory(VfsTestLib BINARY_DIR ${outdir} LIBRARY_DIR ${outdir})
+get_target_property(test_suite_folder FrontendTests FOLDER)
+set_property(TARGET VfsTestLib PROPERTY FOLDER "${test_suite_folder}")
+add_dependencies(FrontendTests VfsTestLib)

unittests/Frontend/CompilerInstanceTest.cpp

 //===- unittests/Frontend/CompilerInstanceTest.cpp - CI tests -------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/CompilerInvocation.h"
+#include "llvm/Config/config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
 using namespace clang;
 
 namespace {
 
TEST(CompilerInstance, DefaultVFSOverlayFromInvocation) {
   Instance.setInvocation(CInvok);
   Instance.createFileManager();
 
   // Check if the virtual file exists which means that our VFS is used by the
   // CompilerInstance.
   ASSERT_TRUE(Instance.getFileManager().getFile("vfs-virtual.file"));
 }
 
+static std::string LibPath(const std::string Name = "VfsTestLib") {
+  const std::vector<testing::internal::string> &Argvs =
+      testing::internal::GetArgvs();
+  const char *Argv0 = Argvs.size() > 0 ? Argvs[0].c_str() : "FrontendTests";
+  llvm::SmallString<256> Buf(llvm::sys::path::parent_path(Argv0));
+  llvm::sys::path::append(Buf, (Name + LTDL_SHLIB_EXT).c_str());
+  return Buf.str();
+}
+
+TEST(CompilerInstance, VFSOverlayLibrary) {
+  // Create a CompilerInvocation that uses this overlay library.
+  const std::string VFSArg = "-ivfsoverlay-lib" + LibPath();
+  const char *Args[] = {"clang", VFSArg.c_str(), "-xc", "-"};
+
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
+      CompilerInstance::createDiagnostics(new DiagnosticOptions());
+
+  std::shared_ptr<CompilerInvocation> CInvok =
+      createInvocationFromCommandLine(Args, Diags);
+
+  if (!CInvok)
+    FAIL() << "could not create compiler invocation";
+
+  CompilerInstance Instance;
+  Instance.setDiagnostics(Diags.get());
+  Instance.setInvocation(CInvok);
+  Instance.createFileManager();
+
+  // Check if the virtual file exists which means that our VFS is used by the
+  // CompilerInstance.
+  ASSERT_TRUE(Instance.getFileManager().getFile("virtual.file"));
+}
+
 } // anonymous namespace

unittests/Frontend/VfsTestLib.cpp

+//===- unittests/Frontend/VfsTestLib.cpp - CI tests -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/VirtualFileSystem.h"
+#include "llvm/ADT/SmallString.h"
+
+#ifdef _WIN32
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+extern "C" EXPORT clang::vfs::FileSystem *__clang_create_vfs() {
+  auto *FS = new clang::vfs::InMemoryFileSystem();
+
+  llvm::SmallString<128> FileName("virtual.file");
+  std::error_code EC = llvm::sys::fs::make_absolute(FileName);
+  if (EC)
+    return nullptr;
+
+  FS->addFile(FileName, (time_t)0, llvm::MemoryBuffer::getMemBuffer("foobar"));
+
+  return FS;
+}