[clang] Mark re-injected tokens appropriately during pragma handling

[kadircet]

Summary

This hides such tokens from TokenWatcher, preventing crashes in clients
trying to match spelled and expanded tokens.

Fixes https://github.com/clangd/clangd/issues/712

[kadircet]

• Add test

[sammccall]

clang/lib/Parse/ParsePragma.cpp

2610

It was surprisingly hard for me to see that the changes here were just marking everything in TokenArray as reinjected.

Could you do that directly with a loop at the end instead? (It's hard to imagine a significant performance difference)

(And the same in all the other modified sites)

2618

as discussed offline, it wasn't clear to me that this was "in time" as we're not trying to affect the behavior of the subsequent PP.Lex call, not the previous one.

Maybe we could extract a function to mark an ArrayRef as reinject with a name that makes this clear.
markAsReinjectedForRelexing() or something...

[kadircet]

• Introduce a helper to mark tokens as reinjected

clang/lib/Parse/ParsePragma.cpp

 
 /// Handle all MS pragmas.  Simply forwards the tokens after inserting
 /// an annotation token.
 void PragmaMSPragma::HandlePragma(Preprocessor &PP,
                                   PragmaIntroducer Introducer, Token &Tok) {
   Token EoF, AnnotTok;
   EoF.startToken();
   EoF.setKind(tok::eof);
+  EoF.setFlag(clang::Token::IsReinjected);
   AnnotTok.startToken();
   AnnotTok.setKind(tok::annot_pragma_ms_pragma);
   AnnotTok.setLocation(Tok.getLocation());
   AnnotTok.setAnnotationEndLoc(Tok.getLocation());
   SmallVector<Token, 8> TokenVector;
   // Suck up all of the tokens before the eod.
   for (; Tok.isNot(tok::eod); PP.Lex(Tok)) {
+    Tok.setFlag(clang::Token::IsReinjected);
     TokenVector.push_back(Tok);
     AnnotTok.setAnnotationEndLoc(Tok.getLocation());
   }
   // Add a sentinel EoF token to the end of the list.
   TokenVector.push_back(EoF);
   // We must allocate this array with new because EnterTokenStream is going to
   // delete it later.
   auto TokenArray = std::make_unique<Token[]>(TokenVector.size());
while (Tok.isNot(tok::eod)) {
     if (Tok.is(tok::l_paren))
       OpenParens++;
     else if (Tok.is(tok::r_paren)) {
       OpenParens--;
       if (OpenParens == 0 && ValueInParens)
         break;
     }
 
+    Tok.setFlag(clang::Token::IsReinjected);
     ValueList.push_back(Tok);
     PP.Lex(Tok);
   }
 
   if (ValueInParens) {
     // Read ')'
     if (Tok.isNot(tok::r_paren)) {
       PP.Diag(Tok.getLocation(), diag::err_expected) << tok::r_paren;
       return true;
     }
     PP.Lex(Tok);
   }
 
   Token EOFTok;
   EOFTok.startToken();
+  EOFTok.setFlag(clang::Token::IsReinjected);
   EOFTok.setKind(tok::eof);
   EOFTok.setLocation(Tok.getLocation());
   ValueList.push_back(EOFTok); // Terminates expression for parsing.
 
   Info.Toks = llvm::makeArrayRef(ValueList).copy(PP.getPreprocessorAllocator());
 
   Info.PragmaName = PragmaName;
   Info.Option = Option;
while (Tok.isNot(tok::eod)) {
       if (Tok.is(tok::l_paren))
         OpenParens++;
       else if (Tok.is(tok::r_paren)) {
         OpenParens--;
         if (OpenParens == 0)
           break;
       }
 
+      Tok.setFlag(clang::Token::IsReinjected);
       AttributeTokens.push_back(Tok);
       PP.Lex(Tok);
     }
 
     if (AttributeTokens.empty()) {
       PP.Diag(Tok.getLocation(), diag::err_pragma_attribute_expected_attribute);
       return;
     }
     if (Tok.isNot(tok::r_paren)) {
       PP.Diag(Tok.getLocation(), diag::err_expected) << tok::r_paren;
       return;
     }
     SourceLocation EndLoc = Tok.getLocation();
     PP.Lex(Tok);
 
     // Terminate the attribute for parsing.
     Token EOFTok;
     EOFTok.startToken();
     EOFTok.setKind(tok::eof);
     EOFTok.setLocation(EndLoc);
+    EOFTok.setFlag(clang::Token::IsReinjected);
     AttributeTokens.push_back(EOFTok);
 
     Info->Tokens =
         llvm::makeArrayRef(AttributeTokens).copy(PP.getPreprocessorAllocator());
   }
 
   if (Tok.isNot(tok::eod))
     PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)