WIP: Frontend: Handle PreprocessorOptions::RemappedFiles in createVFSFromCompilerInvocation

[dexonsmith]

Summary

Instead of threading PreprocessorOptions::RemappedFiles through to the
SourceManager, handle it in createVFSFromCompilerInvocation.

I've also removed some of the now-dead code from SourceManager. I'll
leave unifying ContentCache::OrigEntry and
ContentCache::ContentsEntry for follow ups, since otherwise there'll
be a lot of noise here.

WIP: Currently I have hacked clang/test/PCH/remap-file-from-pch.cpp to
get it to pass, but probably the right thing is to write a prep patch
that reimplements the following fishy code in ASTReader from
6eec06d0d4fa2e108546b422619c6cb32f234e6e (maybe sending a list of files
that should trigger this behaviour through PreprocessorOptions?), not
necessarily keeping the exact same semantics.

// Check if there was a request to override the contents of the file
// that was part of the precompiled header. Overriding such a file
// can lead to problems when lexing using the source locations from the
// PCH.
SourceManager &SM = getSourceManager();
// FIXME: Reject if the overrides are different.
if ((!Overridden && !Transient) && SM.isFileOverridden(File)) {
  if (Complain)
    Error(diag::err_fe_pch_file_overridden, Filename);

  // After emitting the diagnostic, bypass the overriding file to recover
  // (this creates a separate FileEntry).
  File = SM.bypassFileContentsOverride(*File);
  if (!File) {
    F.InputFilesLoaded[ID - 1] = InputFile::getNotFound();
    return InputFile();
  }
}

clang/include/clang/Basic/SourceManager.h

public:
 
   /// True if this file may be transient, that is, if it might not
   /// exist at some later point in time when this content entry is used,
   /// after serialization and deserialization.
   unsigned IsTransient : 1;
 
   mutable unsigned IsBufferInvalid : 1;
 
-  ContentCache(const FileEntry *Ent = nullptr) : ContentCache(Ent, Ent) {}
-
-  ContentCache(const FileEntry *Ent, const FileEntry *contentEnt)
-      : OrigEntry(Ent), ContentsEntry(contentEnt), BufferOverridden(false),
+  ContentCache(const FileEntry *Ent = nullptr)
+      : OrigEntry(Ent), ContentsEntry(Ent), BufferOverridden(false),
         IsFileVolatile(false), IsTransient(false), IsBufferInvalid(false) {}
 
   /// The copy ctor does not allow copies where source object has either
   /// a non-NULL Buffer or SourceLineCache.  Ownership of allocated memory
   /// is not transferred, so this is a logical error.
   ContentCache(const ContentCache &RHS)
       : BufferOverridden(false), IsFileVolatile(false), IsTransient(false),
         IsBufferInvalid(false) {
class SourceManager : public RefCountedBase<SourceManager> {
   bool UserFilesAreVolatile;
 
   /// True if all files read during this compilation should be treated
   /// as transient (may not be present in later compilations using a module
   /// file created from this compilation). Defaults to false.
   bool FilesAreTransient = false;
 
   struct OverriddenFilesInfoTy {
-    /// Files that have been overridden with the contents from another
-    /// file.
-    llvm::DenseMap<const FileEntry *, const FileEntry *> OverriddenFiles;
-
     /// Files that were overridden with a memory buffer.
     llvm::DenseSet<const FileEntry *> OverriddenFilesWithBuffer;
   };
 
   /// Lazily create the object keeping overridden files info, since
   /// it is uncommonly used.
   std::unique_ptr<OverriddenFilesInfoTy> OverriddenFilesInfo;
 
public:
   ///
   /// \param SourceFile the source file whose contents will be overridden.
   ///
   /// \param Buffer the memory buffer whose contents will be used as the
   /// data in the given source file.
   void overrideFileContents(const FileEntry *SourceFile,
                             std::unique_ptr<llvm::MemoryBuffer> Buffer);
 
-  /// Override the given source file with another one.
-  ///
-  /// \param SourceFile the source file which will be overridden.
-  ///
-  /// \param NewFile the file whose contents will be used as the
-  /// data instead of the contents of the given source file.
-  void overrideFileContents(const FileEntry *SourceFile,
-                            const FileEntry *NewFile);
-
   /// Returns true if the file contents have been overridden.
   bool isFileOverridden(const FileEntry *File) const {
     if (OverriddenFilesInfo) {
       if (OverriddenFilesInfo->OverriddenFilesWithBuffer.count(File))
         return true;
-      if (OverriddenFilesInfo->OverriddenFiles.find(File) !=
-          OverriddenFilesInfo->OverriddenFiles.end())
-        return true;
     }
     return false;
   }
 
   /// Bypass the overridden contents of a file.  This creates a new FileEntry
   /// and initializes the content cache for it.  Returns nullptr if there is no
   /// such file in the filesystem.
   ///

clang/lib/Basic/SourceManager.cpp

ContentCache &SourceManager::getOrCreateContentCache(const FileEntry *FileEnt,
   // Do we already have information about this file?
   ContentCache *&Entry = FileInfos[FileEnt];
   if (Entry)
     return *Entry;
 
   // Nope, create a new Cache entry.
   Entry = ContentCacheAlloc.Allocate<ContentCache>();
 
-  if (OverriddenFilesInfo) {
-    // If the file contents are overridden with contents from another file,
-    // pass that file to ContentCache.
-    llvm::DenseMap<const FileEntry *, const FileEntry *>::iterator
-        overI = OverriddenFilesInfo->OverriddenFiles.find(FileEnt);
-    if (overI == OverriddenFilesInfo->OverriddenFiles.end())
-      new (Entry) ContentCache(FileEnt);
-    else
-      new (Entry) ContentCache(OverridenFilesKeepOriginalName ? FileEnt
-                                                              : overI->second,
-                               overI->second);
-  } else {
-    new (Entry) ContentCache(FileEnt);
-  }
 
   Entry->IsFileVolatile = UserFilesAreVolatile && !isSystemFile;
   Entry->IsTransient = FilesAreTransient;
 
   return *Entry;
 }
 
 /// Create a new ContentCache for the specified memory buffer.
void SourceManager::overrideFileContents(
   SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile);
 
   IR.setBuffer(std::move(Buffer));
   IR.BufferOverridden = true;
 
   getOverriddenFilesInfo().OverriddenFilesWithBuffer.insert(SourceFile);
 }
 
-void SourceManager::overrideFileContents(const FileEntry *SourceFile,
-                                         const FileEntry *NewFile) {
-  assert(SourceFile->getSize() == NewFile->getSize() &&
-         "Different sizes, use the FileManager to create a virtual file with "
-         "the correct size");
-  assert(FileInfos.count(SourceFile) == 0 &&
-         "This function should be called at the initialization stage, before "
-         "any parsing occurs.");
-  getOverriddenFilesInfo().OverriddenFiles[SourceFile] = NewFile;
-}
-
 const FileEntry *
 SourceManager::bypassFileContentsOverride(const FileEntry &File) {
   assert(isFileOverridden(&File));
   llvm::Optional<FileEntryRef> BypassFile =
       FileMgr.getBypassFile(File.getLastRef());
 
   // If the file can't be found in the FS, give up.
   if (!BypassFile)

 
 size_t SourceManager::getDataStructureSizes() const {
   size_t size = llvm::capacity_in_bytes(MemBufferInfos)
     + llvm::capacity_in_bytes(LocalSLocEntryTable)
     + llvm::capacity_in_bytes(LoadedSLocEntryTable)
     + llvm::capacity_in_bytes(SLocEntryLoaded)
     + llvm::capacity_in_bytes(FileInfos);
 
-  if (OverriddenFilesInfo)
-    size += llvm::capacity_in_bytes(OverriddenFilesInfo->OverriddenFiles);
-
   return size;
 }
 
 SourceManagerForFile::SourceManagerForFile(StringRef FileName,
                                            StringRef Content) {
   // This is referenced by `FileMgr` and will be released by `FileMgr` when it
   // is deleted.
   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem(

clang/lib/Frontend/ASTUnit.cpp

getBufferForFileHandlingRemapping(const CompilerInvocation &Invocation,
   // command line (to another file) or directly through the compiler
   // invocation (to a memory buffer).
   llvm::MemoryBuffer *Buffer = nullptr;
   std::unique_ptr<llvm::MemoryBuffer> BufferOwner;
   auto FileStatus = VFS->status(FilePath);
   if (FileStatus) {
     llvm::sys::fs::UniqueID MainFileID = FileStatus->getUniqueID();
 
-    // Check whether there is a file-file remapping of the main file
-    for (const auto &RF : PreprocessorOpts.RemappedFiles) {
-      std::string MPath(RF.first);
-      auto MPathStatus = VFS->status(MPath);
-      if (MPathStatus) {
-        llvm::sys::fs::UniqueID MID = MPathStatus->getUniqueID();
-        if (MainFileID == MID) {
-          // We found a remapping. Try to load the resulting, remapped source.
-          BufferOwner = valueOrNull(VFS->getBufferForFile(RF.second, -1, true, isVolatile));
-          if (!BufferOwner)
-            return nullptr;
-        }
-      }
-    }
-
     // Check whether there is a file-buffer remapping. It supercedes the
     // file-file remapping.
     for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
       std::string MPath(RB.first);
       auto MPathStatus = VFS->status(MPath);
       if (MPathStatus) {
         llvm::sys::fs::UniqueID MID = MPathStatus->getUniqueID();
         if (MainFileID == MID) {
std::shared_ptr<CompilerInvocation> CI;
 
     CI = createInvocationFromCommandLine(llvm::makeArrayRef(ArgBegin, ArgEnd),
                                          Diags);
     if (!CI)
       return nullptr;
   }
 
   PreprocessorOptions &PPOpts = CI->getPreprocessorOpts();
-  PPOpts.RemappedFilesKeepOriginalName = true;
   PPOpts.AllowPCHWithCompilerErrors = AllowPCHWithCompilerErrors;
   PPOpts.SingleFileParseMode = SingleFileParse;
   PPOpts.RetainExcludedConditionalBlocks = RetainExcludedConditionalBlocks;
 
   // Override the resources path.
   CI->getHeaderSearchOpts().ResourceDir = std::string(ResourceFilesPath);
 
   CI->getFrontendOpts().SkipFunctionBodies =

clang/lib/Frontend/CompilerInstance.cpp

for (const auto &RB : InitOpts.RemappedFileBuffers) {
     // to the SourceManager.
     if (InitOpts.RetainRemappedFileBuffers)
       SourceMgr.overrideFileContents(FromFile, RB.second->getMemBufferRef());
     else
       SourceMgr.overrideFileContents(
           FromFile, std::unique_ptr<llvm::MemoryBuffer>(
                         const_cast<llvm::MemoryBuffer *>(RB.second)));
   }
-
-  // Remap files in the source manager (with other files).
-  for (const auto &RF : InitOpts.RemappedFiles) {
-    // Find the file that we're mapping to.
-    auto ToFile = FileMgr.getFile(RF.second);
-    if (!ToFile) {
-      Diags.Report(diag::err_fe_remap_missing_to_file) << RF.first << RF.second;
-      continue;
-    }
-
-    // Create the file entry for the file that we're mapping from.
-    const FileEntry *FromFile =
-        FileMgr.getVirtualFile(RF.first, (*ToFile)->getSize(), 0);
-    if (!FromFile) {
-      Diags.Report(diag::err_fe_remap_missing_from_file) << RF.first;
-      continue;
-    }
-
-    // Override the contents of the "from" file with the contents of
-    // the "to" file.
-    SourceMgr.overrideFileContents(FromFile, *ToFile);
-  }
-
-  SourceMgr.setOverridenFilesKeepOriginalName(
-      InitOpts.RemappedFilesKeepOriginalName);
 }
 
 // Preprocessor
 
 void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
   const PreprocessorOptions &PPOpts = getPreprocessorOpts();
 
   // The AST reader holds a reference to the old preprocessor (if any).

clang/lib/Frontend/CompilerInvocation.cpp

if (!FS) {
       continue;
     }
 
     Result = FS;
   }
   return Result;
 }
 
+static IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+addRemappedFiles(const CompilerInvocation &CI,
+                 IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
+  auto &PPOpts = CI.getPreprocessorOpts();
+  if (PPOpts.RemappedFiles.empty())
+    return BaseFS;
+
+  IntrusiveRefCntPtr<llvm::vfs::FileSystem> Result =
+      llvm::vfs::RedirectingFileSystem::create(
+          PPOpts.RemappedFiles,
+          /*UseExternalNames=*/!PPOpts.RemappedFilesKeepOriginalName, *BaseFS)
+          .release();
+  return Result;
+}
+
 IntrusiveRefCntPtr<llvm::vfs::FileSystem>
 clang::createVFSFromCompilerInvocation(
     const CompilerInvocation &CI, DiagnosticsEngine &Diags,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) {
   IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = std::move(BaseFS);
   FS = addVFSOverlays(CI, Diags, std::move(FS));
+  FS = addRemappedFiles(CI, std::move(FS));
   return FS;
 }

clang/lib/Frontend/PrecompiledPreamble.cpp

bool PrecompiledPreamble::CanReuse(
   // The preamble has not changed. We may be able to re-use the precompiled
   // preamble.
 
   // Check that none of the files used by the preamble have changed.
   // First, make a record of those files that have been overridden via
   // remapping or unsaved_files.
   std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
   llvm::StringSet<> OverriddenAbsPaths; // Either by buffers or files.
-  for (const auto &R : PreprocessorOpts.RemappedFiles) {
-    llvm::vfs::Status Status;
-    if (!moveOnNoError(VFS.status(R.second), Status)) {
-      // If we can't stat the file we're remapping to, assume that something
-      // horrible happened.
-      return false;
-    }
-    // If a mapped file was previously missing, then it has changed.
-    llvm::SmallString<128> MappedPath(R.first);
-    if (!VFS.makeAbsolute(MappedPath))
-      OverriddenAbsPaths.insert(MappedPath);
-
-    OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
-        Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
-  }
 
   // OverridenFileBuffers tracks only the files not found in VFS.
   llvm::StringMap<PreambleFileHash> OverridenFileBuffers;
   for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
     const PrecompiledPreamble::PreambleFileHash PreambleHash =
         PreambleFileHash::createForMemoryBuffer(RB.second->getMemBufferRef());
     llvm::vfs::Status Status;
     if (moveOnNoError(VFS.status(RB.first), Status))

clang/test/PCH/remap-file-from-pch.cpp

-//  %clang_cc1 -remap-file "%s;%S/Inputs/remapped-file" -fsyntax-only %s 2>&1 | FileCheck -check-prefix=CHECK-EXIST %s
-
 // RUN: %clang_cc1 -x c++-header %s.h -emit-pch -o %t.pch
-// RUN: not %clang_cc1 %s -include-pch %t.pch -remap-file "%s.h;%s.remap.h" -fsyntax-only 2>&1 | FileCheck %s
+// RUN: not %clang_cc1 %s -include-pch %t.pch -fsyntax-only 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=NO-REMAP
+// RUN: not %clang_cc1 %s -include-pch %t.pch -remap-file "%s.h;%s.remap.h" -fsyntax-only 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=MISMATCH
+//
+// RUN: %clang_cc1 -x c++-header -remap-file "%s.h;%s.remap.h" %s.h -emit-pch -o %t-remapped.pch
+// RUN: %clang_cc1 %s -include-pch %t-remapped.pch -remap-file "%s.h;%s.remap.h" -fsyntax-only 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=REMAP -allow-empty
+// RUN: not %clang_cc1 %s -include-pch %t-remapped.pch -fsyntax-only 2>&1 \
+// RUN:   | FileCheck %s -check-prefix=MISMATCH
 
 const char *str = STR;
 int ge = zool;
 
-// CHECK: file '{{.*[/\\]}}remap-file-from-pch.cpp.h' from the precompiled header has been overridden
-// CHECK: use of undeclared identifier 'zool'
+// MISMATCH: file '{{.*[/\\]}}remap-file-from-pch.cpp.h' has been modified since the precompiled header
+// NO-REMAP: use of undeclared identifier 'zool'
+// REMAP-NOT: use of undeclared identifier