[clang][lex] Copy token literal data when buffer can be grown

[sunho]

Summary

As part with https://discourse.llvm.org/t/rfc-flexible-lexer-buffering-for-handling-incomplete-input-in-interactive-c-c/64180

NOTE: This patch is WIP.

[sunho]

Update

clang/include/clang/Lex/Lexer.h

bool tryConsumeIdentifierUCN(unsigned &CurOffset, unsigned Size,
                                Token &Result);
 
   /// Try to consume an identifier character encoded in UTF-8.
   /// \param CurPtr Points to the start of the (potential) UTF-8 code unit
   ///        sequence. On success, updated to point past the end of it.
   /// \return \c true if a UTF-8 sequence mapping to an acceptable identifier
   ///         character was lexed, \c false otherwise.
   bool tryConsumeIdentifierUTF8Char(unsigned &CurOffset);
+
+  void SetTokString(Token& Tok, StringRef Data);
 };
 
 } // namespace clang
 
 #endif // LLVM_CLANG_LEX_LEXER_H

clang/lib/Lex/Lexer.cpp

if (C == '\\' && tryConsumeIdentifierUCN(CurOffset, Size, Result))
       continue;
     if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurOffset))
       continue;
     // Neither an expected Unicode codepoint nor a UCN.
     break;
   }
 
   const char *IdStart = BufferStart + BufferOffset;
+  unsigned TokLen = CurOffset - BufferOffset;
   FormTokenWithChars(Result, CurOffset, tok::raw_identifier);
-  Result.setRawIdentifierData(IdStart);
+  SetTokString(Result, StringRef(IdStart, TokLen));
 
   // If we are in raw mode, return this identifier raw.  There is no need to
   // look up identifier information or attempt to macro expand it.
   if (LexingRawMode)
     return true;
 
   // Fill in Result.IdentifierInfo and update the token kind,
   // looking up the identifier in the identifier table.
bool Lexer::LexNumericConstant(Token &Result, unsigned CurOffset) {
   // If we have a UCN or UTF-8 character (perhaps in a ud-suffix), continue.
   if (C == '\\' && tryConsumeIdentifierUCN(CurOffset, Size, Result))
     return LexNumericConstant(Result, CurOffset);
   if (!isASCII(C) && tryConsumeIdentifierUTF8Char(CurOffset))
     return LexNumericConstant(Result, CurOffset);
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferStart + BufferOffset;
+  unsigned TokLen = CurOffset - BufferOffset;
   FormTokenWithChars(Result, CurOffset, tok::numeric_constant);
-  Result.setLiteralData(TokStart);
+  SetTokString(Result, StringRef(TokStart, TokLen));
   return true;
 }
 
 /// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes
 /// in C++11, or warn on a ud-suffix in C++98.
 unsigned Lexer::LexUDSuffix(Token &Result, unsigned CurOffset,
                             bool IsStringLiteral) {
   assert(LangOpts.CPlusPlus);
if (LangOpts.CPlusPlus)
     CurOffset = LexUDSuffix(Result, CurOffset, true);
 
   // If a nul character existed in the string, warn about it.
   if (NulCharacter && !isLexingRawMode())
     Diag(*NulCharacter, diag::null_in_char_or_string) << 1;
 
   // Update the location of the token as well as the BufferPtr instance var.
   const char *TokStart = BufferStart + BufferOffset;
+  unsigned TokLen = CurOffset - BufferOffset;
+
   FormTokenWithChars(Result, CurOffset, Kind);
-  Result.setLiteralData(TokStart);
+  SetTokString(Result, StringRef(TokStart, TokLen));
   return true;
 }
 
 /// LexRawStringLiteral - Lex the remainder of a raw string literal, after
 /// having lexed R", LR", u8R", uR", or UR".
 bool Lexer::LexRawStringLiteral(Token &Result, unsigned CurOffset,
                                 tok::TokenKind Kind) {
   // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3:
bool Lexer::LexRawStringLiteral(Token &Result, unsigned CurOffset,
   }
 
   // If we are in C++11, lex the optional ud-suffix.
   if (LangOpts.CPlusPlus)
     CurOffset = LexUDSuffix(Result, CurOffset, true);
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = &BufferStart[BufferOffset];
+  unsigned TokLen = CurOffset - BufferOffset;
   FormTokenWithChars(Result, CurOffset, Kind);
-  Result.setLiteralData(TokStart);
+  SetTokString(Result, StringRef(TokStart, TokLen));
   return true;
 }
 
 /// LexAngledStringLiteral - Lex the remainder of an angled string literal,
 /// after having lexed the '<' character.  This is used for #include filenames.
 bool Lexer::LexAngledStringLiteral(Token &Result, unsigned CurOffset) {
   // Does this string contain the \0 character?
   std::optional<unsigned> NulCharacter = std::nullopt;
bool Lexer::LexAngledStringLiteral(Token &Result, unsigned CurOffset) {
   }
 
   // If a nul character existed in the string, warn about it.
   if (NulCharacter && !isLexingRawMode())
     Diag(*NulCharacter, diag::null_in_char_or_string) << 1;
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = &BufferStart[BufferOffset];
+  unsigned TokLen = CurOffset - BufferOffset;
   FormTokenWithChars(Result, CurOffset, tok::header_name);
-  Result.setLiteralData(TokStart);
+  SetTokString(Result, StringRef(TokStart, TokLen));
   return true;
 }
 
 void Lexer::codeCompleteIncludedFile(unsigned PathStart,
                                      unsigned CompletionPoint, bool IsAngled) {
   // Completion only applies to the filename, after the last slash.
   StringRef PartialPath(BufferStart + PathStart, CompletionPoint - PathStart);
   llvm::StringRef SlashChars = LangOpts.MSVCCompat ? "/\\" : "/";
if (LangOpts.CPlusPlus)
     CurOffset = LexUDSuffix(Result, CurOffset, false);
 
   // If a nul character existed in the character, warn about it.
   if (NulCharacter && !isLexingRawMode())
     Diag(*NulCharacter, diag::null_in_char_or_string) << 0;
 
   // Update the location of token as well as BufferPtr.
   const char *TokStart = BufferStart + BufferOffset;
+  unsigned TokLen = CurOffset - BufferOffset;
   FormTokenWithChars(Result, CurOffset, Kind);
-  Result.setLiteralData(TokStart);
+  SetTokString(Result, StringRef(TokStart, TokLen));
   return true;
 }
 
 /// SkipWhitespace - Efficiently skip over a series of whitespace characters.
 /// Update BufferPtr to point to the next non-whitespace character and return.
 ///
 /// This method forms a token and returns true if KeepWhitespaceMode is enabled.
 bool Lexer::SkipWhitespace(Token &Result, unsigned CurOffset,
static const char *findPlaceholderEnd(const char *CurPtr,
   BufferEnd -= 1; // Scan until the second last character.
   for (; CurPtr != BufferEnd; ++CurPtr) {
     if (CurPtr[0] == '#' && CurPtr[1] == '>')
       return CurPtr + 2;
   }
   return nullptr;
 }
 
+void Lexer::SetTokString(Token& Tok, StringRef Str) {
+  const char* DestPtr = Str.data();
+  if (GrowBuffer) {
+    assert(PP);
+    PP->CreateString(Str, Tok);
+  }
+
+  if (Tok.is(tok::raw_identifier))
+    Tok.setRawIdentifierData(DestPtr);
+  else
+    Tok.setLiteralData(DestPtr);
+}
+
 bool Lexer::lexEditorPlaceholder(Token &Result, unsigned CurOffset) {
   assert(BufferStart[CurOffset - 1] == '<' && BufferStart[CurOffset] == '#' &&
          "Not a placeholder!");
   if (!PP || !PP->getPreprocessorOpts().LexEditorPlaceholders || LexingRawMode)
     return false;
   if (CurOffset + 1 == BufferSize) 
     TryExpandBuffer();
   const char *End = findPlaceholderEnd(BufferStart + CurOffset + 1, BufferStart + BufferSize);
   if (!End)
     return false;
   const char *Start = BufferStart + CurOffset - 1;
+  unsigned TokLen = End-Start;
   if (!LangOpts.AllowEditorPlaceholders)
     Diag(CurOffset - 1, diag::err_placeholder_in_source);
   Result.startToken();
   FormTokenWithChars(Result, End - BufferStart, tok::raw_identifier);
-  Result.setRawIdentifierData(Start);
+  SetTokString(Result, StringRef(Start, TokLen));
   PP->LookUpIdentifierInfo(Result);
   Result.setFlag(Token::IsEditorPlaceholder);
   BufferOffset = End - BufferStart;
   return true;
 }
 
 bool Lexer::isCodeCompletionPoint(unsigned CurOffset) const {
   if (PP && PP->isCodeCompletionEnabled()) {
if (Result.is(tok::raw_identifier)) {
     if (!isLexingRawMode()) {
       IdentifierInfo *II = PP->LookUpIdentifierInfo(Result);
       if (II->isHandleIdentifierCase())
         return PP->HandleIdentifier(Result);
     }
     return true;
   }
   if (Result.isLiteral()) {
-    Result.setLiteralData(TokPtr);
+    SetTokString(Result, StringRef(TokPtr, Result.getLength()));
     return true;
   }
   if (Result.is(tok::colon) &&
       (LangOpts.CPlusPlus || LangOpts.DoubleSquareBracketAttributes)) {
     // Convert consecutive colons to 'tok::coloncolon'.
     if (BufferStart[BufferOffset] == ':') {
       assert(DepDirectives.front().Tokens[NextDepDirectiveTokenIndex].is(
           tok::colon));