Lexer: Update the Lexer to use MemoryBufferRef, NFC

[dexonsmith]

Summary

Update Lexer / Lexer::Lexer to use MemoryBufferRef instead of
MemoryBuffer*. Callers that were acquiring a MemoryBuffer* via
SourceManager::getBuffer were updated, such that if they checked
Invalid they use getBufferOrNone and otherwise getBufferOrFake.

[JDevlieghere]

clang/include/clang/Lex/Lexer.h

145

Should MemoryBufferRefs be passed by value like StringRefs?

[dexonsmith]

clang/include/clang/Lex/Lexer.h

145

I don't think it absolutely needs to be, but it could, it's just two StringRefs. Passing by reference here prevents adding an include unnecessarily.

[dexonsmith]

clang/include/clang/Lex/Lexer.h

145

That said, I don't feel strongly, happy to change to by-value if you think that's better.

[JDevlieghere]

LGTM

clang/include/clang/Lex/Lexer.h

145

Thanks! I was mostly asking from a "policy" perspective, but avoiding an include seems as good of a motivation as any.

clang-tools-extra/clangd/Format.cpp

 /// Append closing brackets )]} to \p Code to make it well-formed.
 /// Clang-format conservatively refuses to format files with unmatched brackets
 /// as it isn't sure where the errors are and so can't correct.
 /// When editing, it's reasonable to assume code before the cursor is complete.
 void closeBrackets(std::string &Code, const format::FormatStyle &Style) {
   SourceManagerForFile FileSM("dummy.cpp", Code);
   auto &SM = FileSM.get();
   FileID FID = SM.getMainFileID();
-  Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style));
+  Lexer Lex(FID, SM.getBufferOrFake(FID), SM,
+            format::getFormattingLangOpts(Style));
   Token Tok;
   std::vector<char> Brackets;
   while (!Lex.LexFromRawLexer(Tok)) {
     switch(Tok.getKind()) {
       case tok::l_paren:
         Brackets.push_back(')');
         break;
       case tok::l_brace:

clang/include/clang/Lex/Lexer.h

 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
 #include <cstdint>
 #include <string>
 
 namespace llvm {
 
-class MemoryBuffer;
+class MemoryBufferRef;
 
 } // namespace llvm
 
 namespace clang {
 
 class DiagnosticBuilder;
 class Preprocessor;
 class SourceManager;
class Lexer : public PreprocessorLexer {
 
   void InitLexer(const char *BufStart, const char *BufPtr, const char *BufEnd);
 
 public:
   /// Lexer constructor - Create a new lexer object for the specified buffer
   /// with the specified preprocessor managing the lexing process.  This lexer
   /// assumes that the associated file buffer and Preprocessor objects will
   /// outlive it, so it doesn't take ownership of either of them.
-  Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP);
+  Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, Preprocessor &PP);
 
   /// Lexer constructor - Create a new raw lexer object.  This object is only
   /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the
   /// text range will outlive it, so it doesn't take ownership of it.
   Lexer(SourceLocation FileLoc, const LangOptions &LangOpts,
         const char *BufStart, const char *BufPtr, const char *BufEnd);
 
   /// Lexer constructor - Create a new raw lexer object.  This object is only
   /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the
   /// text range will outlive it, so it doesn't take ownership of it.
-  Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
+  Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
         const SourceManager &SM, const LangOptions &LangOpts);
 
   Lexer(const Lexer &) = delete;
   Lexer &operator=(const Lexer &) = delete;
 
   /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
   /// _Pragma expansion.  This has a variety of magic semantics that this method
   /// sets up.  It returns a new'd Lexer that must be delete'd when done.

clang/lib/Format/FormatTokenLexer.cpp

FormatTokenLexer::FormatTokenLexer(
     llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
     IdentifierTable &IdentTable)
     : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
       Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
       Style(Style), IdentTable(IdentTable), Keywords(IdentTable),
       Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0),
       FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
       MacroBlockEndRegex(Style.MacroBlockEnd) {
-  Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
+  Lex.reset(new Lexer(ID, SourceMgr.getBufferOrFake(ID), SourceMgr,
                       getFormattingLangOpts(Style)));
   Lex->SetKeepWhitespaceMode(true);
 
   for (const std::string &ForEachMacro : Style.ForEachMacros)
     Macros.insert({&IdentTable.get(ForEachMacro), TT_ForEachMacro});
   for (const std::string &AttributeMacro : Style.AttributeMacros)
     Macros.insert({&IdentTable.get(AttributeMacro), TT_AttributeMacro});
   for (const std::string &StatementMacro : Style.StatementMacros)

clang/lib/Frontend/FrontendAction.cpp

 ///        directive. (We need to do this because the directive will not be
 ///        visited by the preprocessor.)
 static SourceLocation ReadOriginalFileName(CompilerInstance &CI,
                                            std::string &InputFile,
                                            bool IsModuleMap = false) {
   auto &SourceMgr = CI.getSourceManager();
   auto MainFileID = SourceMgr.getMainFileID();
 
-  bool Invalid = false;
-  const auto *MainFileBuf = SourceMgr.getBuffer(MainFileID, &Invalid);
-  if (Invalid)
+  auto MainFileBuf = SourceMgr.getBufferOrNone(MainFileID);
+  if (!MainFileBuf)
     return SourceLocation();
 
   std::unique_ptr<Lexer> RawLexer(
-      new Lexer(MainFileID, MainFileBuf, SourceMgr, CI.getLangOpts()));
+      new Lexer(MainFileID, *MainFileBuf, SourceMgr, CI.getLangOpts()));
 
   // If the first line has the syntax of
   //
   // # NUM "FILENAME"
   //
   // we use FILENAME as the input file name.
   Token T;
   if (RawLexer->LexFromRawLexer(T) || T.getKind() != tok::hash)

clang/lib/Frontend/FrontendActions.cpp

 // Preprocessor Actions
 //===----------------------------------------------------------------------===//
 
 void DumpRawTokensAction::ExecuteAction() {
   Preprocessor &PP = getCompilerInstance().getPreprocessor();
   SourceManager &SM = PP.getSourceManager();
 
   // Start lexing the specified input file.
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID());
+  llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(SM.getMainFileID());
   Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts());
   RawLex.SetKeepWhitespaceMode(true);
 
   Token RawTok;
   RawLex.LexFromRawLexer(RawTok);
   while (RawTok.isNot(tok::eof)) {
     PP.DumpToken(RawTok, true);
     llvm::errs() << "\n";

clang/lib/Frontend/Rewrite/InclusionRewriter.cpp

struct IncludedFile {
     IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType,
                  const DirectoryLookup *DirLookup)
         : Id(Id), FileType(FileType), DirLookup(DirLookup) {}
   };
   Preprocessor &PP; ///< Used to find inclusion directives.
   SourceManager &SM; ///< Used to read and manage source files.
   raw_ostream &OS; ///< The destination stream for rewritten contents.
   StringRef MainEOL; ///< The line ending marker to use.
-  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
+  llvm::MemoryBufferRef PredefinesBuffer; ///< The preprocessor predefines.
   bool ShowLineMarkers; ///< Show #line markers.
   bool UseLineDirectives; ///< Use of line directives or line markers.
   /// Tracks where inclusions that change the file are found.
   std::map<SourceLocation, IncludedFile> FileIncludes;
   /// Tracks where inclusions that import modules are found.
   std::map<SourceLocation, const Module *> ModuleIncludes;
   /// Tracks where inclusions that enter modules (in a module build) are found.
   std::map<SourceLocation, const Module *> ModuleEntryIncludes;
   /// Tracks where #if and #elif directives get evaluated and whether to true.
   std::map<SourceLocation, bool> IfConditions;
   /// Used transitively for building up the FileIncludes mapping over the
   /// various \c PPCallbacks callbacks.
   SourceLocation LastInclusionLocation;
 public:
   InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
                     bool UseLineDirectives);
   void Process(FileID FileId, SrcMgr::CharacteristicKind FileType,
                const DirectoryLookup *DirLookup);
-  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
+  void setPredefinesBuffer(const llvm::MemoryBufferRef &Buf) {
     PredefinesBuffer = Buf;
   }
   void detectMainFileEOL();
   void handleModuleBegin(Token &Tok) {
     assert(Tok.getKind() == tok::annot_module_begin);
     ModuleEntryIncludes.insert(
         {Tok.getLocation(), (Module *)Tok.getAnnotationValue()});
   }
private:
   void If(SourceLocation Loc, SourceRange ConditionRange,
           ConditionValueKind ConditionValue) override;
   void Elif(SourceLocation Loc, SourceRange ConditionRange,
             ConditionValueKind ConditionValue, SourceLocation IfLoc) override;
   void WriteLineInfo(StringRef Filename, int Line,
                      SrcMgr::CharacteristicKind FileType,
                      StringRef Extra = StringRef());
   void WriteImplicitModuleImport(const Module *Mod);
-  void OutputContentUpTo(const MemoryBuffer &FromFile,
-                         unsigned &WriteFrom, unsigned WriteTo,
-                         StringRef EOL, int &lines,
+  void OutputContentUpTo(const MemoryBufferRef &FromFile, unsigned &WriteFrom,
+                         unsigned WriteTo, StringRef EOL, int &lines,
                          bool EnsureNewline);
   void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
-                           const MemoryBuffer &FromFile, StringRef EOL,
+                           const MemoryBufferRef &FromFile, StringRef EOL,
                            unsigned &NextToWrite, int &Lines);
   const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
   const Module *FindModuleAtLocation(SourceLocation Loc) const;
   const Module *FindEnteredModule(SourceLocation Loc) const;
   bool IsIfAtLocationTrue(SourceLocation Loc) const;
   StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
 };
 
 }  // end anonymous namespace
 
 /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
 InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
                                      bool ShowLineMarkers,
                                      bool UseLineDirectives)
     : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
-      PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
-      UseLineDirectives(UseLineDirectives),
+      ShowLineMarkers(ShowLineMarkers), UseLineDirectives(UseLineDirectives),
       LastInclusionLocation(SourceLocation()) {}
 
 /// Write appropriate line information as either #line directives or GNU line
 /// markers depending on what mode we're in, including the \p Filename and
 /// \p Line we are located at, using the specified \p EOL line separator, and
 /// any \p Extra context specifiers in GNU line directives.
 void InclusionRewriter::WriteLineInfo(StringRef Filename, int Line,
                                       SrcMgr::CharacteristicKind FileType,
bool InclusionRewriter::IsIfAtLocationTrue(SourceLocation Loc) const {
   const auto I = IfConditions.find(Loc);
   if (I != IfConditions.end())
     return I->second;
   return false;
 }
 
 /// Detect the likely line ending style of \p FromFile by examining the first
 /// newline found within it.
-static StringRef DetectEOL(const MemoryBuffer &FromFile) {
+static StringRef DetectEOL(const MemoryBufferRef &FromFile) {
   // Detect what line endings the file uses, so that added content does not mix
   // the style. We need to check for "\r\n" first because "\n\r" will match
   // "\r\n\r\n".
   const char *Pos = strchr(FromFile.getBufferStart(), '\n');
   if (!Pos)
     return "\n";
   if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
     return "\r\n";
   if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
     return "\n\r";
   return "\n";
 }
 
 void InclusionRewriter::detectMainFileEOL() {
-  bool Invalid;
-  const MemoryBuffer &FromFile = *SM.getBuffer(SM.getMainFileID(), &Invalid);
-  assert(!Invalid);
-  if (Invalid)
+  Optional<MemoryBufferRef> FromFile = *SM.getBufferOrNone(SM.getMainFileID());
+  assert(FromFile);
+  if (!FromFile)
     return; // Should never happen, but whatever.
-  MainEOL = DetectEOL(FromFile);
+  MainEOL = DetectEOL(*FromFile);
 }
 
 /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
 /// \p WriteTo - 1.
-void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
+void InclusionRewriter::OutputContentUpTo(const MemoryBufferRef &FromFile,
                                           unsigned &WriteFrom, unsigned WriteTo,
                                           StringRef LocalEOL, int &Line,
                                           bool EnsureNewline) {
   if (WriteTo <= WriteFrom)
     return;
-  if (&FromFile == PredefinesBuffer) {
+  if (FromFile == PredefinesBuffer) {
     // Ignore the #defines of the predefines buffer.
     WriteFrom = WriteTo;
     return;
   }
 
   // If we would output half of a line ending, advance one character to output
   // the whole line ending.  All buffers are null terminated, so looking ahead
   // one byte is safe.

 
 /// Print characters from \p FromFile starting at \p NextToWrite up until the
 /// inclusion directive at \p StartToken, then print out the inclusion
 /// inclusion directive disabled by a #if directive, updating \p NextToWrite
 /// and \p Line to track the number of source lines visited and the progress
 /// through the \p FromFile buffer.
 void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
                                             const Token &StartToken,
-                                            const MemoryBuffer &FromFile,
+                                            const MemoryBufferRef &FromFile,
                                             StringRef LocalEOL,
                                             unsigned &NextToWrite, int &Line) {
   OutputContentUpTo(FromFile, NextToWrite,
                     SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
                     false);
   Token DirectiveToken;
   do {
     DirectiveLex.LexFromRawLexer(DirectiveToken);
   } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
-  if (&FromFile == PredefinesBuffer) {
+  if (FromFile == PredefinesBuffer) {
     // OutputContentUpTo() would not output anything anyway.
     return;
   }
   OS << "#if 0 /* expanded by -frewrite-includes */" << MainEOL;
   OutputContentUpTo(FromFile, NextToWrite,
                     SM.getFileOffset(DirectiveToken.getLocation()) +
                         DirectiveToken.getLength(),
                     LocalEOL, Line, true);
StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
   return StringRef();
 }
 
 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
 /// and including content of included files recursively.
 void InclusionRewriter::Process(FileID FileId,
                                 SrcMgr::CharacteristicKind FileType,
                                 const DirectoryLookup *DirLookup) {
-  bool Invalid;
-  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
-  assert(!Invalid && "Attempting to process invalid inclusion");
+  MemoryBufferRef FromFile;
+  {
+    auto B = SM.getBufferOrNone(FileId);
+    assert(B && "Attempting to process invalid inclusion");
+    if (B)
+      FromFile = *B;
+  }
   StringRef FileName = FromFile.getBufferIdentifier();
-  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
+  Lexer RawLex(FileId, FromFile, PP.getSourceManager(), PP.getLangOpts());
   RawLex.SetCommentRetentionState(false);
 
   StringRef LocalEOL = DetectEOL(FromFile);
 
   // Per the GNU docs: "1" indicates entering a new file.
   if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
     WriteLineInfo(FileName, 1, FileType, "");
   else
void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
   // to a mode where it would parse only preprocessor directives and comments,
   // nothing else matters for parsing or processing.
   PP.SetMacroExpansionOnlyInDirectives();
   do {
     PP.Lex(Tok);
     if (Tok.is(tok::annot_module_begin))
       Rewrite->handleModuleBegin(Tok);
   } while (Tok.isNot(tok::eof));
-  Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID()));
+  Rewrite->setPredefinesBuffer(SM.getBufferOrFake(PP.getPredefinesFileID()));
   Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User, nullptr);
   Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User, nullptr);
   OS->flush();
 }

clang/lib/Frontend/Rewrite/RewriteMacros.cpp

 /// LexRawTokensFromMainFile - Lets all the raw tokens from the main file into
 /// the specified vector.
 static void LexRawTokensFromMainFile(Preprocessor &PP,
                                      std::vector<Token> &RawTokens) {
   SourceManager &SM = PP.getSourceManager();
 
   // Create a lexer to lex all the tokens of the main file in raw mode.  Even
   // though it is in raw mode, it will not return comments.
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID());
+  llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(SM.getMainFileID());
   Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts());
 
   // Switch on comment lexing because we really do want them.
   RawLex.SetCommentRetentionState(true);
 
   Token RawTok;
   do {
     RawLex.LexFromRawLexer(RawTok);

clang/lib/Frontend/VerifyDiagnosticConsumer.cpp

 /// \return true if any directives were found.
 static bool findDirectives(SourceManager &SM, FileID FID,
                            const LangOptions &LangOpts) {
   // Create a raw lexer to pull all the comments out of FID.
   if (FID.isInvalid())
     return false;
 
   // Create a lexer to lex all the tokens of the main file in raw mode.
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID);
   Lexer RawLex(FID, FromFile, SM, LangOpts);
 
   // Return comments as tokens, this is how we find expected diagnostics.
   RawLex.SetCommentRetentionState(true);
 
   Token Tok;
   Tok.setKind(tok::comment);
   VerifyDiagnosticConsumer::DirectiveStatus Status =

clang/lib/Lex/Lexer.cpp

 //===- Lexer.cpp - C Language Family Lexer --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements the Lexer and Token interfaces.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/Lexer.h"
 #include "UnicodeCharSets.h"
 #include "clang/Basic/CharInfo.h"
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/MultipleIncludeOpt.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Lex/Token.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/LLVM.h"
-#include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/MathExtras.h"
-#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/NativeFormatting.h"
 #include "llvm/Support/UnicodeCharRanges.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
 #include <string>
void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
 
   NewLinePtr = nullptr;
 }
 
 /// Lexer constructor - Create a new lexer object for the specified buffer
 /// with the specified preprocessor managing the lexing process.  This lexer
 /// assumes that the associated file buffer and Preprocessor objects will
 /// outlive it, so it doesn't take ownership of either of them.
-Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
+Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
+             Preprocessor &PP)
     : PreprocessorLexer(&PP, FID),
       FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
       LangOpts(PP.getLangOpts()) {
-  InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(),
-            InputFile->getBufferEnd());
+  InitLexer(InputFile.getBufferStart(), InputFile.getBufferStart(),
+            InputFile.getBufferEnd());
 
   resetExtendedTokenMode();
 }
 
 /// Lexer constructor - Create a new raw lexer object.  This object is only
 /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
 /// range will outlive it, so it doesn't take ownership of it.
 Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
              const char *BufStart, const char *BufPtr, const char *BufEnd)
     : FileLoc(fileloc), LangOpts(langOpts) {
   InitLexer(BufStart, BufPtr, BufEnd);
 
   // We *are* in raw mode.
   LexingRawMode = true;
 }
 
 /// Lexer constructor - Create a new raw lexer object.  This object is only
 /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
 /// range will outlive it, so it doesn't take ownership of it.
-Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
+Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
              const SourceManager &SM, const LangOptions &langOpts)
-    : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile->getBufferStart(),
-            FromFile->getBufferStart(), FromFile->getBufferEnd()) {}
+    : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(),
+            FromFile.getBufferStart(), FromFile.getBufferEnd()) {}
 
 void Lexer::resetExtendedTokenMode() {
   assert(PP && "Cannot reset token mode without a preprocessor");
   if (LangOpts.TraditionalCPP)
     SetKeepWhitespaceMode(true);
   else
     SetCommentRetentionState(PP->getCommentRetentionState());
 }

 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
                                  SourceLocation ExpansionLocStart,
                                  SourceLocation ExpansionLocEnd,
                                  unsigned TokLen, Preprocessor &PP) {
   SourceManager &SM = PP.getSourceManager();
 
   // Create the lexer as if we were going to lex the file normally.
   FileID SpellingFID = SM.getFileID(SpellingLoc);
-  const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID);
+  llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(SpellingFID);
   Lexer *L = new Lexer(SpellingFID, InputFile, PP);
 
   // Now that the lexer is created, change the start/end locations so that we
   // just lex the subsection of the file that we want.  This is lexing from a
   // scratch buffer.
   const char *StrData = SM.getCharacterData(SpellingLoc);
 
   L->BufferPtr = StrData;

clang/lib/Lex/PPLexerChange.cpp

 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements pieces of the Preprocessor interface that manage the
 // current lexer stack.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/Path.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
 // Miscellaneous Methods.
 //===----------------------------------------------------------------------===//
 
 /// isInPrimaryFile - Return true if we're in the top-level file, not in a
bool Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir,
                                    SourceLocation Loc) {
   assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
   ++NumEnteredSourceFiles;
 
   if (MaxIncludeStackDepth < IncludeMacroStack.size())
     MaxIncludeStackDepth = IncludeMacroStack.size();
 
   // Get the MemoryBuffer for this FID, if it fails, we fail.
-  bool Invalid = false;
-  const llvm::MemoryBuffer *InputFile =
-    getSourceManager().getBuffer(FID, Loc, &Invalid);
-  if (Invalid) {
+  llvm::Optional<llvm::MemoryBufferRef> InputFile =
+      getSourceManager().getBufferOrNone(FID, Loc);
+  if (!InputFile) {
     SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
     Diag(Loc, diag::err_pp_error_opening_file)
         << std::string(SourceMgr.getBufferName(FileStart)) << "";
     return true;
   }
 
   if (isCodeCompletionEnabled() &&
       SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
     CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
     CodeCompletionLoc =
         CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
   }
 
-  EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
+  EnterSourceFileWithLexer(new Lexer(FID, *InputFile, *this), CurDir);
   return false;
 }
 
 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
 ///  and start lexing tokens from it instead of the current buffer.
 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
                                             const DirectoryLookup *CurDir) {
 

clang/lib/Rewrite/HTMLRewrite.cpp

 /// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
 /// information about keywords, macro expansions etc.  This uses the macro
 /// table state from the end of the file, so it won't be perfectly perfect,
 /// but it will be reasonably close.
 void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
   RewriteBuffer &RB = R.getEditBuffer(FID);
 
   const SourceManager &SM = PP.getSourceManager();
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID);
   Lexer L(FID, FromFile, SM, PP.getLangOpts());
   const char *BufferStart = L.getBuffer().data();
 
   // Inform the preprocessor that we want to retain comments as tokens, so we
   // can highlight them.
   L.SetCommentRetentionState(true);
 
   // Lex all the tokens in raw mode, to avoid entering #includes or expanding

 /// file, to re-expand macros and insert (into the HTML) information about the
 /// macro expansions.  This won't be perfectly perfect, but it will be
 /// reasonably close.
 void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
   // Re-lex the raw token stream into a token buffer.
   const SourceManager &SM = PP.getSourceManager();
   std::vector<Token> TokenStream;
 
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID);
   Lexer L(FID, FromFile, SM, PP.getLangOpts());
 
   // Lex all the tokens in raw mode, to avoid entering #includes or expanding
   // macros.
   while (1) {
     Token Tok;
     L.LexFromRawLexer(Tok);
 

clang/lib/Rewrite/TokenRewriter.cpp

 
 using namespace clang;
 
 TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
                              const LangOptions &LangOpts) {
   ScratchBuf.reset(new ScratchBuffer(SM));
 
   // Create a lexer to lex all the tokens of the main file in raw mode.
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID);
   Lexer RawLex(FID, FromFile, SM, LangOpts);
 
   // Return all comments and whitespace as tokens.
   RawLex.SetKeepWhitespaceMode(true);
 
   // Lex the file, populating our datastructures.
   Token RawTok;
   RawLex.LexFromRawLexer(RawTok);

clang/lib/Tooling/Inclusions/HeaderIncludes.cpp

 // \p GetOffsetAfterSequence should be a function that matches a sequence of
 // tokens and returns an offset after the sequence.
 unsigned getOffsetAfterTokenSequence(
     StringRef FileName, StringRef Code, const IncludeStyle &Style,
     llvm::function_ref<unsigned(const SourceManager &, Lexer &, Token &)>
         GetOffsetAfterSequence) {
   SourceManagerForFile VirtualSM(FileName, Code);
   SourceManager &SM = VirtualSM.get();
-  Lexer Lex(SM.getMainFileID(), SM.getBuffer(SM.getMainFileID()), SM,
+  Lexer Lex(SM.getMainFileID(), SM.getBufferOrFake(SM.getMainFileID()), SM,
             createLangOpts());
   Token Tok;
   // Get the first token.
   Lex.LexFromRawLexer(Tok);
   return GetOffsetAfterSequence(SM, Lex, Tok);
 }
 
 // Check if a sequence of tokens is like "#<Name> <raw_identifier>". If it is,