diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -937,6 +937,9 @@ // into the name of a header unit. ANNOTATION(header_unit) +// Annotation for end of input in clang-repl. +ANNOTATION(input_end) + #undef PRAGMA_ANNOTATION #undef ANNOTATION #undef TESTING_KEYWORD diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -692,7 +692,8 @@ bool isEofOrEom() { tok::TokenKind Kind = Tok.getKind(); return Kind == tok::eof || Kind == tok::annot_module_begin || - Kind == tok::annot_module_end || Kind == tok::annot_module_include; + Kind == tok::annot_module_end || Kind == tok::annot_module_include || + Kind == tok::annot_input_end; } /// Checks if the \p Level is valid for use in a fold expression. diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -663,7 +663,8 @@ // them. if (Tok.is(tok::eof) || (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) && - !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end))) + !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) && + !Tok.is(tok::annot_input_end))) return; // EmittedDirectiveOnThisLine takes priority over RequireSameLine. @@ -819,6 +820,9 @@ // -traditional-cpp the lexer keeps /all/ whitespace, including comments. PP.Lex(Tok); continue; + } else if (Tok.is(tok::annot_input_end)) { + PP.Lex(Tok); + continue; } else if (Tok.is(tok::eod)) { // Don't print end of directive tokens, since they are typically newlines // that mess up our line tracking. These come from unknown pre-processor diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp --- a/clang/lib/Interpreter/IncrementalParser.cpp +++ b/clang/lib/Interpreter/IncrementalParser.cpp @@ -158,8 +158,8 @@ LastPTU.TUPart = C.getTranslationUnitDecl(); // Skip previous eof due to last incremental input. - if (P->getCurToken().is(tok::eof)) { - P->ConsumeToken(); + if (P->getCurToken().is(tok::annot_input_end)) { + P->ConsumeAnyToken(); // FIXME: Clang does not call ExitScope on finalizing the regular TU, we // might want to do that around HandleEndOfTranslationUnit. P->ExitScope(); @@ -259,12 +259,12 @@ Token Tok; do { PP.Lex(Tok); - } while (Tok.isNot(tok::eof)); + } while (Tok.isNot(tok::annot_input_end)); } Token AssertTok; PP.Lex(AssertTok); - assert(AssertTok.is(tok::eof) && + assert(AssertTok.is(tok::annot_input_end) && "Lexer must be EOF when starting incremental parse!"); if (CodeGenerator *CG = getCodeGen(Act.get())) { diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp --- a/clang/lib/Lex/PPLexerChange.cpp +++ b/clang/lib/Lex/PPLexerChange.cpp @@ -535,13 +535,19 @@ return LeavingSubmodule; } } - // If this is the end of the main file, form an EOF token. assert(CurLexer && "Got EOF but no current lexer set!"); const char *EndPos = getCurLexerEndPos(); Result.startToken(); CurLexer->BufferPtr = EndPos; - CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); + + if (isIncrementalProcessingEnabled()) { + CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_input_end); + Result.setAnnotationEndLoc(Result.getLocation()); + Result.setAnnotationValue(nullptr); + } else { + CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); + } if (isCodeCompletionEnabled()) { // Inserting the code-completion point increases the source buffer by 1, diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp --- a/clang/lib/Parse/ParseCXXInlineMethods.cpp +++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -836,6 +836,7 @@ case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: + case tok::annot_input_end: // Ran out of tokens. return false; @@ -1242,6 +1243,7 @@ case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: + case tok::annot_input_end: // Ran out of tokens. return false; diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -2061,6 +2061,7 @@ case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: + case tok::annot_input_end: return; default: diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -320,6 +320,7 @@ case tok::annot_module_begin: case tok::annot_module_end: case tok::annot_module_include: + case tok::annot_input_end: // Stop before we change submodules. They generally indicate a "good" // place to pick up parsing again (except in the special case where // we're trying to skip to EOF). @@ -616,8 +617,8 @@ // Skip over the EOF token, flagging end of previous input for incremental // processing - if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof)) - ConsumeToken(); + if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::annot_input_end)) + ConsumeAnyToken(); Result = nullptr; switch (Tok.getKind()) { @@ -697,6 +698,7 @@ return false; case tok::eof: + case tok::annot_input_end: // Check whether -fmax-tokens= was reached. if (PP.getMaxTokens() != 0 && PP.getTokenCount() > PP.getMaxTokens()) { PP.Diag(Tok.getLocation(), diag::warn_max_tokens_total)