[Clang][Lex] Generate an annotation token for clang-repl

[junaire]

Summary

This patches adds a new annotation token called annot_input_end, which
can be used to capture expression in value printing. It works somewhat
like annot_end_of_module token we already have.

This approach is proposed by Richard Smith, thanks!

Signed-off-by: Jun Zhang <[e-mail]>

[junaire]

Merged changes into the child revision.

clang/include/clang/Basic/TokenKinds.def

 ANNOTATION(module_include)
 ANNOTATION(module_begin)
 ANNOTATION(module_end)
 
 // Annotation for a header_name token that has been looked up and transformed
 // into the name of a header unit.
 ANNOTATION(header_unit)
 
+// Annotation for end of input in clang-repl.
+ANNOTATION(input_end)
+
 #undef PRAGMA_ANNOTATION
 #undef ANNOTATION
 #undef TESTING_KEYWORD
 #undef OBJC_AT_KEYWORD
 #undef CXX_KEYWORD_OPERATOR
 #undef PPKEYWORD
 #undef ALIAS
 #undef EXPRESSION_TRAIT

clang/include/clang/Parse/Parser.h

void cutOffParsing() {
     Tok.setKind(tok::eof);
   }
 
   /// Determine if we're at the end of the file or at a transition
   /// between modules.
   bool isEofOrEom() {
     tok::TokenKind Kind = Tok.getKind();
     return Kind == tok::eof || Kind == tok::annot_module_begin ||
-           Kind == tok::annot_module_end || Kind == tok::annot_module_include;
+           Kind == tok::annot_module_end || Kind == tok::annot_module_include ||
+           Kind == tok::annot_input_end;
   }
 
   /// Checks if the \p Level is valid for use in a fold expression.
   bool isFoldOperator(prec::Level Level) const;
 
   /// Checks if the \p Kind is a valid operator for fold expressions.
   bool isFoldOperator(tok::TokenKind Kind) const;
 

clang/lib/Frontend/PrintPreprocessedOutput.cpp

 
 void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(const Token &Tok,
                                                          bool RequireSpace,
                                                          bool RequireSameLine) {
   // These tokens are not expanded to anything and don't need whitespace before
   // them.
   if (Tok.is(tok::eof) ||
       (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) &&
-       !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end)))
+       !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) &&
+       !Tok.is(tok::annot_input_end)))
     return;
 
   // EmittedDirectiveOnThisLine takes priority over RequireSameLine.
   if ((!RequireSameLine || EmittedDirectiveOnThisLine) &&
       MoveToLine(Tok, /*RequireStartOfLine=*/EmittedDirectiveOnThisLine)) {
     if (MinimizeWhitespace) {
       // Avoid interpreting hash as a directive under -fpreprocessed.
       if (Tok.is(tok::hash))
Callbacks->HandleWhitespaceBeforeTok(Tok, /*RequireSpace=*/false,
                                          /*RequireSameLine=*/!IsStartOfLine);
 
     if (DropComments && Tok.is(tok::comment)) {
       // Skip comments. Normally the preprocessor does not generate
       // tok::comment nodes at all when not keeping comments, but under
       // -traditional-cpp the lexer keeps /all/ whitespace, including comments.
       PP.Lex(Tok);
       continue;
+    } else if (Tok.is(tok::annot_input_end)) {
+      PP.Lex(Tok);
+      continue;
     } else if (Tok.is(tok::eod)) {
       // Don't print end of directive tokens, since they are typically newlines
       // that mess up our line tracking. These come from unknown pre-processor
       // directives or hash-prefixed comments in standalone assembly files.
       PP.Lex(Tok);
       // FIXME: The token on the next line after #include should have
       // Tok.isAtStartOfLine() set.
       IsStartOfLine = true;

clang/lib/Interpreter/IncrementalParser.cpp

IncrementalParser::ParseOrWrapTopLevelDecl() {
   PTUs.emplace_back(PartialTranslationUnit());
   PartialTranslationUnit &LastPTU = PTUs.back();
   // Add a new PTU.
   ASTContext &C = S.getASTContext();
   C.addTranslationUnitDecl();
   LastPTU.TUPart = C.getTranslationUnitDecl();
 
   // Skip previous eof due to last incremental input.
-  if (P->getCurToken().is(tok::eof)) {
-    P->ConsumeToken();
+  if (P->getCurToken().is(tok::annot_input_end)) {
+    P->ConsumeAnyToken();
     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
     // might want to do that around HandleEndOfTranslationUnit.
     P->ExitScope();
     S.CurContext = nullptr;
     // Start a new PTU.
     P->EnterScope(Scope::DeclScope);
     S.ActOnTranslationUnitScope(P->getCurScope());
   }
IncrementalParser::Parse(llvm::StringRef input) {
   if (PP.getLangOpts().DelayedTemplateParsing) {
     // Microsoft-specific:
     // Late parsed templates can leave unswallowed "macro"-like tokens.
     // They will seriously confuse the Parser when entering the next
     // source file. So lex until we are EOF.
     Token Tok;
     do {
       PP.Lex(Tok);
-    } while (Tok.isNot(tok::eof));
+    } while (Tok.isNot(tok::annot_input_end));
   }
 
   Token AssertTok;
   PP.Lex(AssertTok);
-  assert(AssertTok.is(tok::eof) &&
+  assert(AssertTok.is(tok::annot_input_end) &&
          "Lexer must be EOF when starting incremental parse!");
 
   if (CodeGenerator *CG = getCodeGen(Act.get())) {
     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
                     M->getContext());
 
     PTU->TheModule = std::move(M);

clang/lib/Lex/PPLexerChange.cpp

if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
       // Leaving the through header. Continue directly to end of main file
       // processing.
       LeavingPCHThroughHeader = true;
     } else {
       // Client should lex another token unless we generated an EOM.
       return LeavingSubmodule;
     }
   }
-
   // If this is the end of the main file, form an EOF token.
   assert(CurLexer && "Got EOF but no current lexer set!");
   const char *EndPos = getCurLexerEndPos();
   Result.startToken();
   CurLexer->BufferPtr = EndPos;
+
+  if (isIncrementalProcessingEnabled()) {
+    CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_input_end);
+    Result.setAnnotationEndLoc(Result.getLocation());
+    Result.setAnnotationValue(nullptr);
+  } else {
     CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
+  }
 
   if (isCodeCompletionEnabled()) {
     // Inserting the code-completion point increases the source buffer by 1,
     // but the main FileID was created before inserting the point.
     // Compensate by reducing the EOF location by 1, otherwise the location
     // will point to the next FileID.
     // FIXME: This is hacky, the code-completion point should probably be
     // inserted before the main FileID is created.

clang/lib/Parse/ParseCXXInlineMethods.cpp

if (Tok.is(T1) || Tok.is(T2)) {
       return true;
     }
 
     switch (Tok.getKind()) {
     case tok::eof:
     case tok::annot_module_begin:
     case tok::annot_module_end:
     case tok::annot_module_include:
+    case tok::annot_input_end:
       // Ran out of tokens.
       return false;
 
     case tok::l_paren:
       // Recursively consume properly-nested parens.
       Toks.push_back(Tok);
       ConsumeParen();
       ConsumeAndStoreUntil(tok::r_paren, Toks, /*StopAtSemi=*/false);
case tok::comma:
       // Keep going. We know we're inside a template argument list now.
       ++KnownTemplateCount;
       goto consume_token;
 
     case tok::eof:
     case tok::annot_module_begin:
     case tok::annot_module_end:
     case tok::annot_module_include:
+    case tok::annot_input_end:
       // Ran out of tokens.
       return false;
 
     case tok::less:
       // FIXME: A '<' can only start a template-id if it's preceded by an
       // identifier, an operator-function-id, or a literal-operator-id.
       ++AngleCount;
       goto consume_token;

clang/lib/Parse/ParseDecl.cpp

case tok::plus:
       if (Tok.isAtStartOfLine() && ParsingInObjCContainer)
         return;
       break;
 
     case tok::eof:
     case tok::annot_module_begin:
     case tok::annot_module_end:
     case tok::annot_module_include:
+    case tok::annot_input_end:
       return;
 
     default:
       break;
     }
 
     ConsumeAnyToken();
   }

clang/lib/Parse/Parser.cpp

case tok::annot_pragma_openmp_end:
       // Stop before an OpenMP pragma boundary.
       if (OpenMPDirectiveParsing)
         return false;
       ConsumeAnnotationToken();
       break;
     case tok::annot_module_begin:
     case tok::annot_module_end:
     case tok::annot_module_include:
+    case tok::annot_input_end:
       // Stop before we change submodules. They generally indicate a "good"
       // place to pick up parsing again (except in the special case where
       // we're trying to skip to EOF).
       return false;
 
     case tok::code_completion:
       if (!HasFlagsSet(Flags, StopAtCodeCompletion))
         handleUnexpectedCodeCompletionToken();

 ///           declaration
 /// [C++20]   module-import-declaration
 bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result,
                                Sema::ModuleImportState &ImportState) {
   DestroyTemplateIdAnnotationsRAIIObj CleanupRAII(*this);
 
   // Skip over the EOF token, flagging end of previous input for incremental
   // processing
-  if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof))
-    ConsumeToken();
+  if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::annot_input_end))
+    ConsumeAnyToken();
 
   Result = nullptr;
   switch (Tok.getKind()) {
   case tok::annot_pragma_unused:
     HandlePragmaUnused();
     return false;
 
   case tok::kw_export:
bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result,
   case tok::annot_module_end:
     Actions.ActOnModuleEnd(Tok.getLocation(), reinterpret_cast<Module *>(
                                                   Tok.getAnnotationValue()));
     ConsumeAnnotationToken();
     ImportState = Sema::ModuleImportState::NotACXX20Module;
     return false;
 
   case tok::eof:
+  case tok::annot_input_end:
     // Check whether -fmax-tokens= was reached.
     if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) {
       PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total)
           << PP.getTokenCount() << PP.getMaxTokens();
       SourceLocation OverrideLoc = PP.getMaxTokensOverrideLoc();
       if (OverrideLoc.isValid()) {
         PP.Diag(OverrideLoc, diag::note_max_tokens_total_override);
       }