[clang][lex] Add TryGrowLexerBuffer/SourceFileGrower

[sunho]

Summary

Add TryGrowLexerBuffer to Lexer and SourceFileGrower to Preprocessor that can be used to grow the source code buffer when Lexer reaches the eof of the buffer.

Since clang-repl receive source code incrementally line by line, we need to grow source code buffer. This change adds the interfaces to Lexer/Preprocessor in order to accommodate this need. If SourceFileGrower interface is registered to Preprocessor, it will call TryGrowFile method when Lexer reached eof. Inside this method, the user can grow the file and return true to request Lexer to continue the lexing from the last point.

When Lexer reaches eof it will call TryGrowLexerBuffer callback specified in the constructor to try growing buffer and if it got new buffer, it will continue the lexing from the last point. Preprocessor registers TryGrowLexerBuffer callback in order to implement handling of SourceFileGrower.

Note that practically all the code changes will not affect the AOT clang world since they are all disabled when Preprocessor doesn't have SourceFileGrower instance.

NOTE: This is part 2 and 3 of https://discourse.llvm.org/t/rfc-flexible-lexer-buffering-for-handling-incomplete-input-in-interactive-c-c/64180

The debian failure is due to some clang-format issue we seems to be unrelatd to this change.

[sunho]

Update

[sunho]

Update

[sunho]

Update

clang/include/clang/Lex/Lexer.h

 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_LEX_LEXER_H
 #define LLVM_CLANG_LEX_LEXER_H
 
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TokenKinds.h"
+#include "llvm/ADT/FunctionExtras.h"
 #include "clang/Lex/DependencyDirectivesScanner.h"
 #include "clang/Lex/PreprocessorLexer.h"
 #include "clang/Lex/Token.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
 #include <cstdint>
 #include <optional>
class Lexer : public PreprocessorLexer {
   bool HasLeadingEmptyMacro;
 
   /// True if this is the first time we're lexing the input file.
   bool IsFirstTimeLexingFile;
 
   // NewLineOffset - A offset to new line character '\n' being lexed. For
   // '\r\n', it also points to '\n.'
   Optional<unsigned> NewLineOffset;
+  using GrowBufferCallback = llvm::unique_function<std::optional<llvm::MemoryBufferRef>()>;
+  GrowBufferCallback GrowBuffer;
 
   // CurrentConflictMarkerState - The kind of conflict marker we are handling.
   ConflictMarkerKind CurrentConflictMarkerState;
 
   /// Non-empty if this \p Lexer is \p isDependencyDirectivesLexer().
   ArrayRef<dependency_directives_scan::Directive> DepDirectives;
 
   /// If this \p Lexer is \p isDependencyDirectivesLexer(), it represents the
   /// next token to use from the current dependency directive.
   unsigned NextDepDirectiveTokenIndex = 0;
 
   void InitLexer(const char *BufStart, unsigned BufferOffset,
                  unsigned BufferSize);
 
 public:
   /// Lexer constructor - Create a new lexer object for the specified buffer
   /// with the specified preprocessor managing the lexing process.  This lexer
   /// assumes that the associated file buffer and Preprocessor objects will
   /// outlive it, so it doesn't take ownership of either of them.
   Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile, Preprocessor &PP,
-        bool IsFirstIncludeOfFile = true);
+        bool IsFirstIncludeOfFile = true, GrowBufferCallback GrowBuffer = nullptr);
 
   /// Lexer constructor - Create a new raw lexer object.  This object is only
   /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the
   /// text range will outlive it, so it doesn't take ownership of it.
   Lexer(SourceLocation FileLoc, const LangOptions &LangOpts,
         const char *BufStart, const char *BufPtr, const char *BufEnd,
         bool IsFirstIncludeOfFile = true);
 
   /// Lexer constructor - Create a new raw lexer object.  This object is only
   /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the
   /// text range will outlive it, so it doesn't take ownership of it.
   Lexer(FileID FID, const llvm::MemoryBufferRef &FromFile,
         const SourceManager &SM, const LangOptions &LangOpts,
         bool IsFirstIncludeOfFile = true);
 
   Lexer(const Lexer &) = delete;
   Lexer &operator=(const Lexer &) = delete;
 
+  bool TryExpandBuffer();
+
   /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for
   /// _Pragma expansion.  This has a variety of magic semantics that this method
   /// sets up.  It returns a new'd Lexer that must be delete'd when done.
   static Lexer *Create_PragmaLexer(SourceLocation SpellingLoc,
                                    SourceLocation ExpansionLocStart,
                                    SourceLocation ExpansionLocEnd,
                                    unsigned TokLen, Preprocessor &PP);
 

clang/lib/Lex/Lexer.cpp

void Lexer::InitLexer(const char *BufStart, unsigned BufOffset,
   NewLineOffset = llvm::None;
 }
 
 /// Lexer constructor - Create a new lexer object for the specified buffer
 /// with the specified preprocessor managing the lexing process.  This lexer
 /// assumes that the associated file buffer and Preprocessor objects will
 /// outlive it, so it doesn't take ownership of either of them.
 Lexer::Lexer(FileID FID, const llvm::MemoryBufferRef &InputFile,
-             Preprocessor &PP, bool IsFirstIncludeOfFile)
+             Preprocessor &PP, bool IsFirstIncludeOfFile, GrowBufferCallback GrowBuffer)
     : PreprocessorLexer(&PP, FID),
       FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)),
       LangOpts(PP.getLangOpts()), LineComment(LangOpts.LineComment),
-      IsFirstTimeLexingFile(IsFirstIncludeOfFile) {
-  InitLexer(InputFile.getBufferStart(), 0, InputFile.getBufferSize());
+      IsFirstTimeLexingFile(IsFirstIncludeOfFile), 
+      GrowBuffer(std::move(GrowBuffer)) {
+  InitLexer(InputFile.getBufferStart(), 0,
+            InputFile.getBufferSize());
 
   resetExtendedTokenMode();
 }
 
 /// Lexer constructor - Create a new raw lexer object.  This object is only
 /// suitable for calls to 'LexFromRawLexer'.  This lexer assumes that the text
 /// range will outlive it, so it doesn't take ownership of it.
 Lexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts,
if (!Tok.needsCleaning()) {
     Buffer = TokStart;
     return Tok.getLength();
   }
 
   // Otherwise, hard case, relex the characters into the string.
   return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer));
 }
 
+ bool Lexer::TryExpandBuffer() {
+  if (!GrowBuffer)
+    return false;
+  
+  auto NewBuffer = GrowBuffer();
+  if (!NewBuffer)
+    return false;
+
+  BufferStart = NewBuffer->getBufferStart();
+  BufferSize = NewBuffer->getBufferSize();
+  return true;
+}
+
 /// MeasureTokenLength - Relex the token at the specified location and return
 /// its length in bytes in the input file.  If the token needs cleaning (e.g.
 /// includes a trigraph or an escaped newline) then this count includes bytes
 /// that are part of that.
 unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
                                    const SourceManager &SM,
                                    const LangOptions &LangOpts) {
   Token TheTok;
if (unsigned EscapedNewLineSize =
       if (BufferStart[Offset] != '\n' && BufferStart[Offset] != '\r' && Tok &&
           !isLexingRawMode())
         Diag(Offset, diag::backslash_newline_space);
 
       // Found backslash<whitespace><newline>.  Parse the char after it.
       Size += EscapedNewLineSize;
       Offset += EscapedNewLineSize;
 
+      if (BufferStart[Offset] == 0 && Offset == BufferSize) 
+        TryExpandBuffer();
+
+      if (BufferStart[Offset] == 0 && Offset == BufferSize) 
+        TryExpandBuffer();
+
       // Use slow version to accumulate a correct size field.
       return getCharAndSizeSlow(Offset, Size, Tok);
     }
 
     // Otherwise, this is not an escaped newline, just return the slash.
     return '\\';
 }
 
if (C == '\'') {
     return true;
   }
 
   while (C != '\'') {
     // Skip escaped characters.
     if (C == '\\')
       C = getAndAdvanceChar(CurOffset, Result);
 
-    if (C == '\n' || C == '\r' ||                  // Newline.
-        (C == 0 && CurOffset - 1 == BufferSize)) { // End of file.
+    if (C == '\n' || C == '\r' ||             // Newline.
+        (C == 0 && CurOffset-1 == BufferSize && !TryExpandBuffer())) {  // End of file.
       if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
         Diag(BufferOffset, diag::ext_unterminated_char_or_string) << 0;
       FormTokenWithChars(Result, CurOffset - 1, tok::unknown);
       return true;
     }
 
     if (C == 0) {
       if (isCodeCompletionPoint(CurOffset - 1)) {
if (SawNewline)
     setLastNewLine(CurOffset - 1);
 
   // Skip consecutive spaces efficiently.
   while (true) {
     // Skip horizontal whitespace very aggressively.
     while (isHorizontalWhitespace(Char))
       Char = BufferStart[++CurOffset];
 
+    if (Char == 0 && CurOffset == BufferSize+1 && TryExpandBuffer()) {
+      --CurOffset;
+      continue;
+    }
+
     // Otherwise if we have something other than whitespace, we're done.
     if (!isVerticalWhitespace(Char))
       break;
 
     if (ParsingPreprocessorDirective) {
       // End of preprocessor directive line, let LexTokenInternal handle this.
       BufferOffset = CurOffset;
       return false;
bool Lexer::SkipLineComment(Token &Result, unsigned CurOffset,
   // diagnostic only once per entire ill-formed subsequence to avoid
   // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
   bool UnicodeDecodingAlreadyDiagnosed = false;
 
   char C;
   while (true) {
     C = BufferStart[CurOffset];
     // Skip over characters in the fast loop.
+    while (true) {
       while (isASCII(C) && C != 0 &&   // Potentially EOF.
             C != '\n' && C != '\r') { // Newline or DOS-style newline.
         C = BufferStart[++CurOffset];
         UnicodeDecodingAlreadyDiagnosed = false;
       }
+      if (C != 0 || CurOffset != BufferSize + 1 || !TryExpandBuffer()) break;
+      C = BufferStart[CurOffset];
+    }
 
     if (!isASCII(C)) {
       unsigned Length = llvm::getUTF8SequenceSize(
           (const llvm::UTF8 *)&BufferStart[CurOffset],
           (const llvm::UTF8 *)&BufferStart[BufferSize]);
       if (Length == 0) {
         if (!UnicodeDecodingAlreadyDiagnosed && !isLexingRawMode())
           Diag(CurOffset, diag::warn_invalid_utf8_in_comment);
if (CurOffset != OldOffset + 1 && C != '/' &&
           }
 
           if (!isLexingRawMode())
             Diag(OldOffset - 1, diag::ext_multi_line_line_comment);
           break;
         }
     }
 
-    if (C == '\r' || C == '\n' || CurOffset == BufferSize + 1) {
+    if (CurOffset == BufferSize + 1) {
+      if (!TryExpandBuffer()) {
+        --CurOffset;
+        break;
+      }
+      continue;
+    }
+
+    if (C == '\r' || C == '\n') {
       --CurOffset;
       break;
     }
 
     if (C == '\0' && isCodeCompletionPoint(CurOffset - 1)) {
       PP->CodeCompleteNaturalLanguage();
       cutOffLexing();
       return false;
bool Lexer::SkipBlockComment(Token &Result, unsigned CurOffset,
   // comments.
 
   // The first character we get with newlines and trigraphs skipped to handle
   // the degenerate /*/ case below correctly if the * has an escaped newline
   // after it.
   unsigned CharSize;
   unsigned char C = getCharAndSize(CurOffset, CharSize);
   CurOffset += CharSize;
-  if (C == 0 && CurOffset == BufferSize + 1) {
+  if (C == 0 && CurOffset == BufferSize+1 && !TryExpandBuffer()) {
     if (!isLexingRawMode())
       Diag(BufferOffset, diag::err_unterminated_block_comment);
     --CurOffset;
 
     // KeepWhitespaceMode should return this broken comment as a token.  Since
     // it isn't a well formed comment, just return it as an 'unknown' token.
     if (isKeepWhitespaceMode()) {
       FormTokenWithChars(Result, CurOffset, tok::unknown);
bool Lexer::SkipBlockComment(Token &Result, unsigned CurOffset,
 
   // C++23 [lex.phases] p1
   // Diagnose invalid UTF-8 if the corresponding warning is enabled, emitting a
   // diagnostic only once per entire ill-formed subsequence to avoid
   // emiting to many diagnostics (see http://unicode.org/review/pr-121.html).
   bool UnicodeDecodingAlreadyDiagnosed = false;
 
   while (true) {
+    if (CurOffset + 24 >= BufferSize) {
+      TryExpandBuffer();
+    }
     // Skip over all non-interesting characters until we find end of buffer or a
     // (probably ending) '/' character.
     if (CurOffset + 24 < BufferSize &&
         // If there is a code-completion point avoid the fast scan because it
         // doesn't check for '\0'.
         !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) {
       // While not aligned to a 16-byte boundary.
       while (C != '/' && (intptr_t)(BufferStart + CurOffset) % 16 != 0) {
FoundSlash:
       }
       if (BufferStart[CurOffset] == '*' && BufferStart[CurOffset + 1] != '/') {
         // If this is a /* inside of the comment, emit a warning.  Don't do this
         // if this is a /*/, which will end the comment.  This misses cases with
         // embedded escaped newlines, but oh well.
         if (!isLexingRawMode())
           Diag(CurOffset - 1, diag::warn_nested_block_comment);
       }
-  } else if (C == 0 && CurOffset == BufferSize + 1) {
+    } else if (C == 0 && CurOffset == BufferSize+1 && TryExpandBuffer()) {
+      --CurOffset;
+      continue;
+    } else if (C == 0 && CurOffset == BufferSize+1) {
       if (!isLexingRawMode())
         Diag(BufferOffset, diag::err_unterminated_block_comment);
       // Note: the user probably forgot a */.  We could continue immediately
       // after the /*, but this would involve lexing a lot of what really is the
       // comment, which surely would confuse the parser.
       --CurOffset;
 
       // KeepWhitespaceMode should return this broken comment as a token.  Since
void Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) {
   char Char = getAndAdvanceChar(CurOffset, Tmp);
   switch (Char) {
   default:
       if (Result)
         Result->push_back(Char);
       break;
     case 0:  // Null.
       // Found end of file?
-      if (CurOffset - 1 != BufferSize) {
-        if (isCodeCompletionPoint(CurOffset - 1)) {
+      if (CurOffset-1 != BufferSize && !TryExpandBuffer()) {
+        if (isCodeCompletionPoint(CurOffset-1)) {
           PP->CodeCompleteNaturalLanguage();
           cutOffLexing();
           return;
         }
 
         // Nope, normal character, continue.
         if (Result)
           Result->push_back(Char);
static const char *findPlaceholderEnd(const char *CurPtr,
   return nullptr;
 }
 
 bool Lexer::lexEditorPlaceholder(Token &Result, unsigned CurOffset) {
   assert(BufferStart[CurOffset - 1] == '<' && BufferStart[CurOffset] == '#' &&
          "Not a placeholder!");
   if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
     return false;
-  const char *End =
-      findPlaceholderEnd(BufferStart + CurOffset + 1, BufferStart + BufferSize);
+  if (CurOffset + 1 == BufferSize) 
+    TryExpandBuffer();
+  const char *End = findPlaceholderEnd(BufferStart + CurOffset + 1, BufferStart + BufferSize);
   if (!End)
     return false;
   const char *Start = BufferStart + CurOffset - 1;
   if (!LangOpts.AllowEditorPlaceholders)
     Diag(CurOffset - 1, diag::err_placeholder_in_source);
   Result.startToken();
   FormTokenWithChars(Result, End - BufferStart, tok::raw_identifier);
   Result.setRawIdentifierData(Start);
LexStart:
   assert(!Result.hasPtrData() && "Result has not been reset");
 
   // CurPtr - Cache BufferPtr in an automatic variable.
   unsigned CurOffset = BufferOffset;
 
   // Small amounts of horizontal whitespace is very common between tokens.
   if (isHorizontalWhitespace(BufferStart[CurOffset])) {
     do {
+      do {
         ++CurOffset;
       } while (isHorizontalWhitespace(BufferStart[CurOffset]));
+    } while (BufferStart[CurOffset] == 0 && CurOffset == BufferSize && TryExpandBuffer());
 
     // If we are keeping whitespace and other tokens, just return what we just
     // skipped.  The next lexer invocation will return the token after the
     // whitespace.
     if (isKeepWhitespaceMode()) {
       FormTokenWithChars(Result, CurOffset, tok::unknown);
       // FIXME: The next token will not have LeadingSpace set.
       return true;
LexStart:
   tok::TokenKind Kind;
 
   if (!isVerticalWhitespace(Char))
     NewLineOffset = llvm::None;
 
   switch (Char) {
   case 0:  // Null.
     // Found end of file?
-    if (CurOffset - 1 == BufferSize)
-      return LexEndOfFile(Result, CurOffset - 1);
+    if (CurOffset-1 == BufferSize) {
+      if (TryExpandBuffer()) {
+        goto LexNextToken;
+      }
+      return LexEndOfFile(Result, CurOffset-1);
+    }
 
     // Check if we are performing code completion.
     if (isCodeCompletionPoint(CurOffset - 1)) {
       // Return the code-completion token.
       Result.startToken();
       FormTokenWithChars(Result, CurOffset, tok::code_completion);
       return true;
     }