Fix for expanding __pragmas in macro arguments

[akhuang]

Summary

Avoid executing __pragma when macro arguments are pre-expanded.
This is what clang currently does when parsing _Pragmas.

Fixes https://bugs.llvm.org/show_bug.cgi?id=41128, where clang crashes
when trying to get the length of annot_pragma tokens.

[akhuang]

comments/whitespace/test

[rnk]

Nice, the solution was right in front of us the whole time. :)

lgtm, but please share the TokenCollector class.

clang/lib/Lex/Pragma.cpp

168

Let's pull this class out to file scope (and put it in an anonymous namespace) so we can reuse it instead of copying it.

[akhuang]

• move TokenCollector out of function

[rnk]

+ @Bigcheese, for amusement. We discussed some interesting behavior of _Pragma and C++ raw string literals last night.

clang/lib/Lex/Pragma.cpp

void Preprocessor::Handle_Pragma(Token &Tok) {
   //
   // Case #2 appears to be a wording bug: only _Pragmas that would survive to
   // the end of phase 4 should actually be executed. Discussion on the WG14
   // mailing list suggests that a _Pragma operator is notionally checked early,
   // but only pragmas that survive to the end of phase 4 should be executed.
   //
   // In Case #2, we check the syntax now, but then put the tokens back into the
   // token stream for later consumption.
-
   struct TokenCollector {
     Preprocessor &Self;
     bool Collect;
     SmallVector<Token, 3> Tokens;
     Token &Tok;
 
     void lex() {
       if (Collect)
         Tokens.push_back(Tok);
void revert() {
       Self.EnterTokenStream(std::move(Toks), Tokens.size(),
                             /*DisableMacroExpansion*/ true,
                             /*IsReinject*/ true);
 
       // ... and return the _Pragma token unchanged.
       Tok = *Tokens.begin();
     }
   };
-
   TokenCollector Toks = {*this, InMacroArgPreExpansion, {}, Tok};
 
   // Remember the pragma token location.
   SourceLocation PragmaLoc = Tok.getLocation();
 
   // Read the '('.
   Toks.lex();
   if (Tok.isNot(tok::l_paren)) {
void Preprocessor::Handle_Pragma(Token &Tok) {
 
   // Finally, return whatever came after the pragma directive.
   return Lex(Tok);
 }
 
 /// HandleMicrosoft__pragma - Like Handle_Pragma except the pragma text
 /// is not enclosed within a string literal.
 void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
+  struct TokenCollector {
+    Preprocessor &Self;
+    bool Collect;
+    SmallVector<Token, 32> Tokens;
+    Token &Tok;
+
+    void lex() {
+      if (Collect)
+        Tokens.push_back(Tok);
+      Self.Lex(Tok);
+    }
+
+    void revert() {
+      assert(Collect && "did not collect tokens");
+      assert(!Tokens.empty() && "collected unexpected number of tokens");
+
+      // Push the pragma content tokens into the token stream.
+      auto Toks = std::make_unique<Token[]>(Tokens.size());
+      std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get());
+      Toks[Tokens.size() - 1] = Tok;
+      Self.EnterTokenStream(std::move(Toks), Tokens.size(),
+                            /*DisableMacroExpansion*/ true,
+                            /*IsReinject*/ true);
+
+      // ... and return the _Pragma token unchanged.
+      Tok = *Tokens.begin();
+    }
+  };
+  TokenCollector Toks = {*this, InMacroArgPreExpansion, {}, Tok};
+
   // Remember the pragma token location.
   SourceLocation PragmaLoc = Tok.getLocation();
 
   // Read the '('.
-  Lex(Tok);
+  Toks.lex();
   if (Tok.isNot(tok::l_paren)) {
     Diag(PragmaLoc, diag::err__Pragma_malformed);
     return;
   }
 
   // Get the tokens enclosed within the __pragma(), as well as the final ')'.
   SmallVector<Token, 32> PragmaToks;
   int NumParens = 0;
-  Lex(Tok);
+  Toks.lex();
   while (Tok.isNot(tok::eof)) {
     PragmaToks.push_back(Tok);
     if (Tok.is(tok::l_paren))
       NumParens++;
     else if (Tok.is(tok::r_paren) && NumParens-- == 0)
       break;
-    Lex(Tok);
+    Toks.lex();
   }
 
   if (Tok.is(tok::eof)) {
     Diag(PragmaLoc, diag::err_unterminated___pragma);
     return;
   }
 
+  // If we're expanding a macro argument, put the tokens back.
+  if (InMacroArgPreExpansion) {
+    Toks.revert();
+    return;
+  }
+
   PragmaToks.front().setFlag(Token::LeadingSpace);
 
   // Replace the ')' with an EOD to mark the end of the pragma.
   PragmaToks.back().setKind(tok::eod);
 
   Token *TokArray = new Token[PragmaToks.size()];
   std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray);
 

clang/test/Preprocessor/pragma_microsoft.c

 
 #define MACRO_WITH__PRAGMA { \
   __pragma(warning(push)); \
   __pragma(warning(disable: 10000)); \
   1 + (2 > 3) ? 4 : 5; \
   __pragma(warning(pop)); \
 }
 
+#define __PRAGMA_IN_ARG(p) p
+
 void f()
 {
   __pragma() // expected-warning{{unknown pragma ignored}}
 // CHECK: #pragma
 
   // If we ever actually *support* __pragma(warning(disable: x)),
   // this warning should go away.
   MACRO_WITH__PRAGMA // expected-warning {{lower precedence}} \
                      // expected-note 2 {{place parentheses}}
 // CHECK: #pragma warning(push)
 // CHECK: #pragma warning(disable: 10000)
 // CHECK: ; 1 + (2 > 3) ? 4 : 5;
 // CHECK: #pragma warning(pop)
+
+  // Check that __pragma can be in a macro argument.
+  __PRAGMA_IN_ARG(__pragma(pack()))
+// CHECK: #pragma pack()
 }
 
 
 // This should include macro_arg_directive even though the include
 // is looking for test.h  This allows us to assign to "n"
 #pragma include_alias("test.h", "macro_arg_directive.h" )
 #include "test.h"
 void test( void ) {