diff --git a/clang-tools-extra/clangd/Format.cpp b/clang-tools-extra/clangd/Format.cpp --- a/clang-tools-extra/clangd/Format.cpp +++ b/clang-tools-extra/clangd/Format.cpp @@ -26,7 +26,8 @@ SourceManagerForFile FileSM("dummy.cpp", Code); auto &SM = FileSM.get(); FileID FID = SM.getMainFileID(); - Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style)); + Lexer Lex(FID, SM.getBufferOrFake(FID), SM, + format::getFormattingLangOpts(Style)); Token Tok; std::vector Brackets; while (!Lex.LexFromRawLexer(Tok)) { diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -27,7 +27,7 @@ namespace llvm { -class MemoryBuffer; +class MemoryBufferRef; } // namespace llvm @@ -142,7 +142,7 @@ /// with the specified preprocessor managing the lexing process. This lexer /// assumes that the associated file buffer and Preprocessor objects will /// outlive it, so it doesn't take ownership of either of them. - Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP); + Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, Preprocessor &PP); /// Lexer constructor - Create a new raw lexer object. This object is only /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the @@ -153,7 +153,7 @@ /// Lexer constructor - Create a new raw lexer object. This object is only /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the /// text range will outlive it, so it doesn't take ownership of it. - Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, + Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile, const SourceManager &SM, const LangOptions &LangOpts); Lexer(const Lexer &) = delete; diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp --- a/clang/lib/Format/FormatTokenLexer.cpp +++ b/clang/lib/Format/FormatTokenLexer.cpp @@ -33,7 +33,7 @@ Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0), FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin), MacroBlockEndRegex(Style.MacroBlockEnd) { - Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr, + Lex.reset(new Lexer(ID, SourceMgr.getBufferOrFake(ID), SourceMgr, getFormattingLangOpts(Style))); Lex->SetKeepWhitespaceMode(true); diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -233,13 +233,12 @@ auto &SourceMgr = CI.getSourceManager(); auto MainFileID = SourceMgr.getMainFileID(); - bool Invalid = false; - const auto *MainFileBuf = SourceMgr.getBuffer(MainFileID, &Invalid); - if (Invalid) + auto MainFileBuf = SourceMgr.getBufferOrNone(MainFileID); + if (!MainFileBuf) return SourceLocation(); std::unique_ptr RawLexer( - new Lexer(MainFileID, MainFileBuf, SourceMgr, CI.getLangOpts())); + new Lexer(MainFileID, *MainFileBuf, SourceMgr, CI.getLangOpts())); // If the first line has the syntax of // diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -750,7 +750,7 @@ SourceManager &SM = PP.getSourceManager(); // Start lexing the specified input file. - const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID()); + llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(SM.getMainFileID()); Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts()); RawLex.SetKeepWhitespaceMode(true); diff --git a/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp b/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp --- a/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp +++ b/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp @@ -40,7 +40,7 @@ SourceManager &SM; ///< Used to read and manage source files. raw_ostream &OS; ///< The destination stream for rewritten contents. StringRef MainEOL; ///< The line ending marker to use. - const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines. + llvm::MemoryBufferRef PredefinesBuffer; ///< The preprocessor predefines. bool ShowLineMarkers; ///< Show #line markers. bool UseLineDirectives; ///< Use of line directives or line markers. /// Tracks where inclusions that change the file are found. @@ -59,7 +59,7 @@ bool UseLineDirectives); void Process(FileID FileId, SrcMgr::CharacteristicKind FileType, const DirectoryLookup *DirLookup); - void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) { + void setPredefinesBuffer(const llvm::MemoryBufferRef &Buf) { PredefinesBuffer = Buf; } void detectMainFileEOL(); @@ -88,12 +88,11 @@ SrcMgr::CharacteristicKind FileType, StringRef Extra = StringRef()); void WriteImplicitModuleImport(const Module *Mod); - void OutputContentUpTo(const MemoryBuffer &FromFile, - unsigned &WriteFrom, unsigned WriteTo, - StringRef EOL, int &lines, + void OutputContentUpTo(const MemoryBufferRef &FromFile, unsigned &WriteFrom, + unsigned WriteTo, StringRef EOL, int &lines, bool EnsureNewline); void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken, - const MemoryBuffer &FromFile, StringRef EOL, + const MemoryBufferRef &FromFile, StringRef EOL, unsigned &NextToWrite, int &Lines); const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const; const Module *FindModuleAtLocation(SourceLocation Loc) const; @@ -109,8 +108,7 @@ bool ShowLineMarkers, bool UseLineDirectives) : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"), - PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers), - UseLineDirectives(UseLineDirectives), + ShowLineMarkers(ShowLineMarkers), UseLineDirectives(UseLineDirectives), LastInclusionLocation(SourceLocation()) {} /// Write appropriate line information as either #line directives or GNU line @@ -260,7 +258,7 @@ /// Detect the likely line ending style of \p FromFile by examining the first /// newline found within it. -static StringRef DetectEOL(const MemoryBuffer &FromFile) { +static StringRef DetectEOL(const MemoryBufferRef &FromFile) { // Detect what line endings the file uses, so that added content does not mix // the style. We need to check for "\r\n" first because "\n\r" will match // "\r\n\r\n". @@ -275,23 +273,22 @@ } void InclusionRewriter::detectMainFileEOL() { - bool Invalid; - const MemoryBuffer &FromFile = *SM.getBuffer(SM.getMainFileID(), &Invalid); - assert(!Invalid); - if (Invalid) + Optional FromFile = *SM.getBufferOrNone(SM.getMainFileID()); + assert(FromFile); + if (!FromFile) return; // Should never happen, but whatever. - MainEOL = DetectEOL(FromFile); + MainEOL = DetectEOL(*FromFile); } /// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at /// \p WriteTo - 1. -void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile, +void InclusionRewriter::OutputContentUpTo(const MemoryBufferRef &FromFile, unsigned &WriteFrom, unsigned WriteTo, StringRef LocalEOL, int &Line, bool EnsureNewline) { if (WriteTo <= WriteFrom) return; - if (&FromFile == PredefinesBuffer) { + if (FromFile == PredefinesBuffer) { // Ignore the #defines of the predefines buffer. WriteFrom = WriteTo; return; @@ -338,7 +335,7 @@ /// through the \p FromFile buffer. void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex, const Token &StartToken, - const MemoryBuffer &FromFile, + const MemoryBufferRef &FromFile, StringRef LocalEOL, unsigned &NextToWrite, int &Line) { OutputContentUpTo(FromFile, NextToWrite, @@ -348,7 +345,7 @@ do { DirectiveLex.LexFromRawLexer(DirectiveToken); } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof)); - if (&FromFile == PredefinesBuffer) { + if (FromFile == PredefinesBuffer) { // OutputContentUpTo() would not output anything anyway. return; } @@ -376,11 +373,15 @@ void InclusionRewriter::Process(FileID FileId, SrcMgr::CharacteristicKind FileType, const DirectoryLookup *DirLookup) { - bool Invalid; - const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid); - assert(!Invalid && "Attempting to process invalid inclusion"); + MemoryBufferRef FromFile; + { + auto B = SM.getBufferOrNone(FileId); + assert(B && "Attempting to process invalid inclusion"); + if (B) + FromFile = *B; + } StringRef FileName = FromFile.getBufferIdentifier(); - Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts()); + Lexer RawLex(FileId, FromFile, PP.getSourceManager(), PP.getLangOpts()); RawLex.SetCommentRetentionState(false); StringRef LocalEOL = DetectEOL(FromFile); @@ -557,7 +558,7 @@ if (Tok.is(tok::annot_module_begin)) Rewrite->handleModuleBegin(Tok); } while (Tok.isNot(tok::eof)); - Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID())); + Rewrite->setPredefinesBuffer(SM.getBufferOrFake(PP.getPredefinesFileID())); Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User, nullptr); Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User, nullptr); OS->flush(); diff --git a/clang/lib/Frontend/Rewrite/RewriteMacros.cpp b/clang/lib/Frontend/Rewrite/RewriteMacros.cpp --- a/clang/lib/Frontend/Rewrite/RewriteMacros.cpp +++ b/clang/lib/Frontend/Rewrite/RewriteMacros.cpp @@ -64,7 +64,7 @@ // Create a lexer to lex all the tokens of the main file in raw mode. Even // though it is in raw mode, it will not return comments. - const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID()); + llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(SM.getMainFileID()); Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts()); // Switch on comment lexing because we really do want them. diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp --- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp +++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp @@ -827,7 +827,7 @@ return false; // Create a lexer to lex all the tokens of the main file in raw mode. - const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); + llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID); Lexer RawLex(FID, FromFile, SM, LangOpts); // Return comments as tokens, this is how we find expected diagnostics. diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -13,7 +13,9 @@ #include "clang/Lex/Lexer.h" #include "UnicodeCharSets.h" #include "clang/Basic/CharInfo.h" +#include "clang/Basic/Diagnostic.h" #include "clang/Basic/IdentifierTable.h" +#include "clang/Basic/LLVM.h" #include "clang/Basic/LangOptions.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" @@ -24,19 +26,16 @@ #include "clang/Lex/Preprocessor.h" #include "clang/Lex/PreprocessorOptions.h" #include "clang/Lex/Token.h" -#include "clang/Basic/Diagnostic.h" -#include "clang/Basic/LLVM.h" -#include "clang/Basic/TokenKinds.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/NativeFormatting.h" #include "llvm/Support/UnicodeCharRanges.h" #include @@ -133,12 +132,13 @@ /// with the specified preprocessor managing the lexing process. This lexer /// assumes that the associated file buffer and Preprocessor objects will /// outlive it, so it doesn't take ownership of either of them. -Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) +Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, + Preprocessor &PP) : PreprocessorLexer(&PP, FID), FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), LangOpts(PP.getLangOpts()) { - InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), - InputFile->getBufferEnd()); + InitLexer(InputFile.getBufferStart(), InputFile.getBufferStart(), + InputFile.getBufferEnd()); resetExtendedTokenMode(); } @@ -158,10 +158,10 @@ /// Lexer constructor - Create a new raw lexer object. This object is only /// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text /// range will outlive it, so it doesn't take ownership of it. -Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, +Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile, const SourceManager &SM, const LangOptions &langOpts) - : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile->getBufferStart(), - FromFile->getBufferStart(), FromFile->getBufferEnd()) {} + : Lexer(SM.getLocForStartOfFile(FID), langOpts, FromFile.getBufferStart(), + FromFile.getBufferStart(), FromFile.getBufferEnd()) {} void Lexer::resetExtendedTokenMode() { assert(PP && "Cannot reset token mode without a preprocessor"); @@ -194,7 +194,7 @@ // Create the lexer as if we were going to lex the file normally. FileID SpellingFID = SM.getFileID(SpellingLoc); - const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); + llvm::MemoryBufferRef InputFile = SM.getBufferOrFake(SpellingFID); Lexer *L = new Lexer(SpellingFID, InputFile, PP); // Now that the lexer is created, change the start/end locations so that we diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp --- a/clang/lib/Lex/PPLexerChange.cpp +++ b/clang/lib/Lex/PPLexerChange.cpp @@ -11,16 +11,16 @@ // //===----------------------------------------------------------------------===// -#include "clang/Lex/Preprocessor.h" -#include "clang/Lex/PreprocessorOptions.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/LexDiagnostic.h" #include "clang/Lex/MacroInfo.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FileSystem.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/Path.h" using namespace clang; @@ -73,10 +73,9 @@ MaxIncludeStackDepth = IncludeMacroStack.size(); // Get the MemoryBuffer for this FID, if it fails, we fail. - bool Invalid = false; - const llvm::MemoryBuffer *InputFile = - getSourceManager().getBuffer(FID, Loc, &Invalid); - if (Invalid) { + llvm::Optional InputFile = + getSourceManager().getBufferOrNone(FID, Loc); + if (!InputFile) { SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID); Diag(Loc, diag::err_pp_error_opening_file) << std::string(SourceMgr.getBufferName(FileStart)) << ""; @@ -90,7 +89,7 @@ CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset); } - EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir); + EnterSourceFileWithLexer(new Lexer(FID, *InputFile, *this), CurDir); return false; } diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp --- a/clang/lib/Rewrite/HTMLRewrite.cpp +++ b/clang/lib/Rewrite/HTMLRewrite.cpp @@ -445,7 +445,7 @@ RewriteBuffer &RB = R.getEditBuffer(FID); const SourceManager &SM = PP.getSourceManager(); - const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); + llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID); Lexer L(FID, FromFile, SM, PP.getLangOpts()); const char *BufferStart = L.getBuffer().data(); @@ -536,7 +536,7 @@ const SourceManager &SM = PP.getSourceManager(); std::vector TokenStream; - const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); + llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID); Lexer L(FID, FromFile, SM, PP.getLangOpts()); // Lex all the tokens in raw mode, to avoid entering #includes or expanding diff --git a/clang/lib/Rewrite/TokenRewriter.cpp b/clang/lib/Rewrite/TokenRewriter.cpp --- a/clang/lib/Rewrite/TokenRewriter.cpp +++ b/clang/lib/Rewrite/TokenRewriter.cpp @@ -28,7 +28,7 @@ ScratchBuf.reset(new ScratchBuffer(SM)); // Create a lexer to lex all the tokens of the main file in raw mode. - const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); + llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(FID); Lexer RawLex(FID, FromFile, SM, LangOpts); // Return all comments and whitespace as tokens. diff --git a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp --- a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp +++ b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp @@ -43,7 +43,7 @@ GetOffsetAfterSequence) { SourceManagerForFile VirtualSM(FileName, Code); SourceManager &SM = VirtualSM.get(); - Lexer Lex(SM.getMainFileID(), SM.getBuffer(SM.getMainFileID()), SM, + Lexer Lex(SM.getMainFileID(), SM.getBufferOrFake(SM.getMainFileID()), SM, createLangOpts()); Token Tok; // Get the first token.