Basic: Add native support for stdin to SourceManager and FileManager

[dexonsmith]

Summary

Add support for stdin to SourceManager and FileManager. Adds
FileManager::getSTDIN, which adds a FileEntryRef for <stdin> and reads
the MemoryBuffer, which is stored as FileEntry::Content.

Eventually the other buffers in ContentCache will sink to here as well

• we probably usually want to load/save a MemoryBuffer eagerly -- but

it's happening early for stdin to get rid of
CompilerInstance::InitializeSourceManager's final call to
SourceManager::overrideFileContents.

clang/test/CXX/modules-ts/dcl.dcl/dcl.module/dcl.module.export/p1.cpp
relies on building a module from stdin; supporting that requires setting
ContentCache::BufferOverridden.

[arphaman]

Nice, it looks cleaner than the override of file contents that was used before. Left one NIT comment

clang/lib/Basic/FileManager.cpp

352

NIT: It might be better to replace the hardcoded string literal to Content->getBufferIdentifier(), just in case it changes in the future.

clang/include/clang/Basic/FileEntry.h

 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem/UniqueID.h"
 
 namespace llvm {
+
+class MemoryBuffer;
+
 namespace vfs {
 
 class File;
 
 } // namespace vfs
 } // namespace llvm
 
 namespace clang {
public:
   }
   DirectoryEntryRef getDir() const { return *ME->second->Dir; }
 
   inline bool isValid() const;
   inline off_t getSize() const;
   inline unsigned getUID() const;
   inline const llvm::sys::fs::UniqueID &getUniqueID() const;
   inline time_t getModificationTime() const;
+  inline bool isNamedPipe() const;
   inline void closeFile() const;
 
   /// Check if the underlying FileEntry is the same, intentially ignoring
   /// whether the file was referenced with the same spelling of the filename.
   friend bool operator==(const FileEntryRef &LHS, const FileEntryRef &RHS) {
     return &LHS.getFileEntry() == &RHS.getFileEntry();
   }
   friend bool operator==(const FileEntry *LHS, const FileEntryRef &RHS) {
class FileEntry {
   llvm::sys::fs::UniqueID UniqueID;
   unsigned UID = 0; // A unique (small) ID for the file.
   bool IsNamedPipe = false;
   bool IsValid = false; // Is this \c FileEntry initialized and valid?
 
   /// The open file, if it is owned by the \p FileEntry.
   mutable std::unique_ptr<llvm::vfs::File> File;
 
+  /// The file content, if it is owned by the \p FileEntry.
+  std::unique_ptr<llvm::MemoryBuffer> Content;
+
   // First access name for this FileEntry.
   //
   // This is Optional only to allow delayed construction (FileEntryRef has no
   // default constructor). It should always have a value in practice.
   //
   // TODO: remove this once everyone that needs a name uses FileEntryRef.
   Optional<FileEntryRef> LastRef;
 

 const llvm::sys::fs::UniqueID &FileEntryRef::getUniqueID() const {
   return getFileEntry().getUniqueID();
 }
 
 time_t FileEntryRef::getModificationTime() const {
   return getFileEntry().getModificationTime();
 }
 
+bool FileEntryRef::isNamedPipe() const { return getFileEntry().isNamedPipe(); }
+
 void FileEntryRef::closeFile() const { getFileEntry().closeFile(); }
 
 } // end namespace clang
 
 #endif // LLVM_CLANG_BASIC_FILEENTRY_H

clang/include/clang/Basic/FileManager.h

class FileManager : public RefCountedBase<FileManager> {
 
   /// A mirror of SeenFileEntries to give fake answers for getBypassFile().
   ///
   /// Don't bother hooking up a BumpPtrAllocator. This should be rarely used,
   /// and only on error paths.
   std::unique_ptr<llvm::StringMap<llvm::ErrorOr<FileEntryRef::MapValue>>>
       SeenBypassFileEntries;
 
+  /// The file entry for stdin, if it has been accessed through the FileManager.
+  Optional<FileEntryRef> STDIN;
+
   /// The canonical names of files and directories .
   llvm::DenseMap<const void *, llvm::StringRef> CanonicalNames;
 
   /// Storage for canonical names that we have computed.
   llvm::BumpPtrAllocator CanonicalNameStorage;
 
   /// Each FileEntry we create is assigned a unique ID #.
   ///
public:
   /// \param OpenFile if true and the file exists, it will be opened.
   ///
   /// \param CacheFailure If true and the file does not exist, we'll cache
   /// the failure to find this file.
   llvm::Expected<FileEntryRef> getFileRef(StringRef Filename,
                                           bool OpenFile = false,
                                           bool CacheFailure = true);
 
+  /// Get the FileEntryRef for stdin, returning an error if stdin cannot be
+  /// read.
+  ///
+  /// This reads and caches stdin before returning. Subsequent calls return the
+  /// same file entry, and a reference to the cached input is returned by calls
+  /// to getBufferForFile.
+  llvm::Expected<FileEntryRef> getSTDIN();
+
   /// Get a FileEntryRef if it exists, without doing anything on error.
   llvm::Optional<FileEntryRef> getOptionalFileRef(StringRef Filename,
                                                   bool OpenFile = false,
                                                   bool CacheFailure = true) {
     return llvm::expectedToOptional(
         getFileRef(Filename, OpenFile, CacheFailure));
   }
 

clang/lib/Basic/FileEntry.cpp

 //===- FileEntry.cpp - File references --------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
 /// Defines implementation for clang::FileEntry and clang::FileEntryRef.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/FileEntry.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 
 using namespace clang;
 
 FileEntry::FileEntry() : UniqueID(0, 0) {}
 
 FileEntry::~FileEntry() = default;
 
 void FileEntry::closeFile() const { File.reset(); }

clang/lib/Basic/FileManager.cpp

if (auto PathName = UFE.File->getName())
       fillRealPathName(&UFE, *PathName);
   } else if (!openFile) {
     // We should still fill the path even if we aren't opening the file.
     fillRealPathName(&UFE, InterndFileName);
   }
   return ReturnedRef;
 }
 
+llvm::Expected<FileEntryRef> FileManager::getSTDIN() {
+  // Only read stdin once.
+  if (STDIN)
+    return *STDIN;
+
+  std::unique_ptr<llvm::MemoryBuffer> Content;
+  if (auto ContentOrError = llvm::MemoryBuffer::getSTDIN())
+    Content = std::move(*ContentOrError);
+  else
+    return llvm::errorCodeToError(ContentOrError.getError());
+
+  STDIN = getVirtualFileRef(Content->getBufferIdentifier(),
+                            Content->getBufferSize(), 0);
+  FileEntry &FE = const_cast<FileEntry &>(STDIN->getFileEntry());
+  FE.Content = std::move(Content);
+  FE.IsNamedPipe = true;
+  return *STDIN;
+}
+
 const FileEntry *FileManager::getVirtualFile(StringRef Filename, off_t Size,
                                              time_t ModificationTime) {
   return &getVirtualFileRef(Filename, Size, ModificationTime).getFileEntry();
 }
 
 FileEntryRef FileManager::getVirtualFileRef(StringRef Filename, off_t Size,
                                             time_t ModificationTime) {
   ++NumFileLookups;
void FileManager::fillRealPathName(FileEntry *UFE, llvm::StringRef FileName) {
   makeAbsolutePath(AbsPath);
   llvm::sys::path::remove_dots(AbsPath, /*remove_dot_dot=*/true);
   UFE->RealPathName = std::string(AbsPath.str());
 }
 
 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
 FileManager::getBufferForFile(const FileEntry *Entry, bool isVolatile,
                               bool RequiresNullTerminator) {
+  // If the content is living on the file entry, return a reference to it.
+  if (Entry->Content)
+    return llvm::MemoryBuffer::getMemBuffer(Entry->Content->getMemBufferRef());
+
   uint64_t FileSize = Entry->getSize();
   // If there's a high enough chance that the file have changed since we
   // got its size, force a stat before opening it.
   if (isVolatile || Entry->isNamedPipe())
     FileSize = -1;
 
   StringRef Filename = Entry->getName();
   // If the file is already open, use the open file descriptor.

clang/lib/Basic/SourceManager.cpp

else
                                                               : overI->second,
                                overI->second);
   } else {
     new (Entry) ContentCache(FileEnt);
   }
 
   Entry->IsFileVolatile = UserFilesAreVolatile && !isSystemFile;
   Entry->IsTransient = FilesAreTransient;
+  Entry->BufferOverridden |= FileEnt->isNamedPipe();
 
   return *Entry;
 }
 
 /// Create a new ContentCache for the specified memory buffer.
 /// This does no caching.
 ContentCache &SourceManager::createMemBufferContentCache(
     std::unique_ptr<llvm::MemoryBuffer> Buffer) {

clang/lib/Frontend/CompilerInstance.cpp

if (Input.isBuffer()) {
     assert(SourceMgr.getMainFileID().isValid() &&
            "Couldn't establish MainFileID!");
     return true;
   }
 
   StringRef InputFile = Input.getFile();
 
   // Figure out where to get and map in the main file.
-  if (InputFile != "-") {
-    auto FileOrErr = FileMgr.getFileRef(InputFile, /*OpenFile=*/true);
-    if (!FileOrErr) {
-      // FIXME: include the error in the diagnostic.
-      consumeError(FileOrErr.takeError());
+  auto FileOrErr = InputFile == "-"
+                       ? FileMgr.getSTDIN()
+                       : FileMgr.getFileRef(InputFile, /*OpenFile=*/true);
+    // FIXME: include the error in the diagnostic even when it's not stdin.
+    auto EC = llvm::errorToErrorCode(FileOrErr.takeError());
+    if (InputFile != "-")
       Diags.Report(diag::err_fe_error_reading) << InputFile;
-      return false;
-    }
-
-    SourceMgr.setMainFileID(
-        SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind));
-  } else {
-    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> SBOrErr =
-        llvm::MemoryBuffer::getSTDIN();
-    if (std::error_code EC = SBOrErr.getError()) {
+    else
       Diags.Report(diag::err_fe_error_reading_stdin) << EC.message();
-      return false;
-    }
-    std::unique_ptr<llvm::MemoryBuffer> SB = std::move(SBOrErr.get());
 
-    FileEntryRef File = FileMgr.getVirtualFileRef(SB->getBufferIdentifier(),
-                                                  SB->getBufferSize(), 0);
-    SourceMgr.setMainFileID(
-        SourceMgr.createFileID(File, SourceLocation(), Kind));
-    SourceMgr.overrideFileContents(File, std::move(SB));
-  }
+      SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind));
 
   assert(SourceMgr.getMainFileID().isValid() &&
          "Couldn't establish MainFileID!");
   return true;
 }
 
 // High-Level Operations