Fix for expanding __pragmas in macro arguments

[akhuang]

Summary

Avoid executing __pragma when macro arguments are pre-expanded.
This is what clang currently does when parsing _Pragmas.

Fixes https://bugs.llvm.org/show_bug.cgi?id=41128, where clang crashes
when trying to get the length of annot_pragma tokens.

[akhuang]

comments/whitespace/test

[rnk]

Nice, the solution was right in front of us the whole time. :)

lgtm, but please share the TokenCollector class.

clang/lib/Lex/Pragma.cpp

202–203

Let's pull this class out to file scope (and put it in an anonymous namespace) so we can reuse it instead of copying it.

[akhuang]

• move TokenCollector out of function

[rnk]

+ @Bigcheese, for amusement. We discussed some interesting behavior of _Pragma and C++ raw string literals last night.

clang/lib/Lex/Pragma.cpp

void PragmaNamespace::HandlePragma(Preprocessor &PP,
   // Otherwise, pass it down.
   Handler->HandlePragma(PP, Introducer, Tok);
 }
 
 //===----------------------------------------------------------------------===//
 // Preprocessor Pragma Directive Handling.
 //===----------------------------------------------------------------------===//
 
+namespace {
+// TokenCollector provides the option to collect tokens that were "read"
+// and return them to the stream to be read later.
+// Currently used when reading _Pragma/__pragma directives.
+struct TokenCollector {
+  Preprocessor &Self;
+  bool Collect;
+  SmallVector<Token, 3> Tokens;
+  Token &Tok;
+
+  void lex() {
+    if (Collect)
+      Tokens.push_back(Tok);
+    Self.Lex(Tok);
+  }
+
+  void revert() {
+    assert(Collect && "did not collect tokens");
+    assert(!Tokens.empty() && "collected unexpected number of tokens");
+
+    // Push the ( "string" ) tokens into the token stream.
+    auto Toks = std::make_unique<Token[]>(Tokens.size());
+    std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get());
+    Toks[Tokens.size() - 1] = Tok;
+    Self.EnterTokenStream(std::move(Toks), Tokens.size(),
+                          /*DisableMacroExpansion*/ true,
+                          /*IsReinject*/ true);
+
+    // ... and return the pragma token unchanged.
+    Tok = *Tokens.begin();
+  }
+};
+} // namespace
+
 /// HandlePragmaDirective - The "\#pragma" directive has been parsed.  Lex the
 /// rest of the pragma, passing it to the registered pragma handlers.
 void Preprocessor::HandlePragmaDirective(PragmaIntroducer Introducer) {
   if (Callbacks)
     Callbacks->PragmaDirective(Introducer.Loc, Introducer.Kind);
 
   if (!PragmasEnabled)
     return;
void Preprocessor::Handle_Pragma(Token &Tok) {
   //
   // Case #2 appears to be a wording bug: only _Pragmas that would survive to
   // the end of phase 4 should actually be executed. Discussion on the WG14
   // mailing list suggests that a _Pragma operator is notionally checked early,
   // but only pragmas that survive to the end of phase 4 should be executed.
   //
   // In Case #2, we check the syntax now, but then put the tokens back into the
   // token stream for later consumption.
 
-  struct TokenCollector {
-    Preprocessor &Self;
-    bool Collect;
-    SmallVector<Token, 3> Tokens;
-    Token &Tok;
-
-    void lex() {
-      if (Collect)
-        Tokens.push_back(Tok);
-      Self.Lex(Tok);
-    }
-
-    void revert() {
-      assert(Collect && "did not collect tokens");
-      assert(!Tokens.empty() && "collected unexpected number of tokens");
-
-      // Push the ( "string" ) tokens into the token stream.
-      auto Toks = std::make_unique<Token[]>(Tokens.size());
-      std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get());
-      Toks[Tokens.size() - 1] = Tok;
-      Self.EnterTokenStream(std::move(Toks), Tokens.size(),
-                            /*DisableMacroExpansion*/ true,
-                            /*IsReinject*/ true);
-
-      // ... and return the _Pragma token unchanged.
-      Tok = *Tokens.begin();
-    }
-  };
-
   TokenCollector Toks = {*this, InMacroArgPreExpansion, {}, Tok};
 
   // Remember the pragma token location.
   SourceLocation PragmaLoc = Tok.getLocation();
 
   // Read the '('.
   Toks.lex();
   if (Tok.isNot(tok::l_paren)) {
     Diag(PragmaLoc, diag::err__Pragma_malformed);
void Preprocessor::Handle_Pragma(Token &Tok) {
 
   // Finally, return whatever came after the pragma directive.
   return Lex(Tok);
 }
 
 /// HandleMicrosoft__pragma - Like Handle_Pragma except the pragma text
 /// is not enclosed within a string literal.
 void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
+  // During macro pre-expansion, check the syntax now but put the tokens back
+  // into the token stream for later consumption. Same as Handle_Pragma.
+  TokenCollector Toks = {*this, InMacroArgPreExpansion, {}, Tok};
+
   // Remember the pragma token location.
   SourceLocation PragmaLoc = Tok.getLocation();
 
   // Read the '('.
-  Lex(Tok);
+  Toks.lex();
   if (Tok.isNot(tok::l_paren)) {
     Diag(PragmaLoc, diag::err__Pragma_malformed);
     return;
   }
 
   // Get the tokens enclosed within the __pragma(), as well as the final ')'.
   SmallVector<Token, 32> PragmaToks;
   int NumParens = 0;
-  Lex(Tok);
+  Toks.lex();
   while (Tok.isNot(tok::eof)) {
     PragmaToks.push_back(Tok);
     if (Tok.is(tok::l_paren))
       NumParens++;
     else if (Tok.is(tok::r_paren) && NumParens-- == 0)
       break;
-    Lex(Tok);
+    Toks.lex();
   }
 
   if (Tok.is(tok::eof)) {
     Diag(PragmaLoc, diag::err_unterminated___pragma);
     return;
   }
 
+  // If we're expanding a macro argument, put the tokens back.
+  if (InMacroArgPreExpansion) {
+    Toks.revert();
+    return;
+  }
+
   PragmaToks.front().setFlag(Token::LeadingSpace);
 
   // Replace the ')' with an EOD to mark the end of the pragma.
   PragmaToks.back().setKind(tok::eod);
 
   Token *TokArray = new Token[PragmaToks.size()];
   std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray);
 

clang/test/Preprocessor/pragma_microsoft.c

 
 #define MACRO_WITH__PRAGMA { \
   __pragma(warning(push)); \
   __pragma(warning(disable: 10000)); \
   1 + (2 > 3) ? 4 : 5; \
   __pragma(warning(pop)); \
 }
 
+#define PRAGMA_IN_ARGS(p) p
+
 void f()
 {
   __pragma() // expected-warning{{unknown pragma ignored}}
 // CHECK: #pragma
 
   // If we ever actually *support* __pragma(warning(disable: x)),
   // this warning should go away.
   MACRO_WITH__PRAGMA // expected-warning {{lower precedence}} \
                      // expected-note 2 {{place parentheses}}
 // CHECK: #pragma warning(push)
 // CHECK: #pragma warning(disable: 10000)
 // CHECK: ; 1 + (2 > 3) ? 4 : 5;
 // CHECK: #pragma warning(pop)
-}
 
+  // Check that macro arguments can contain __pragma.
+  PRAGMA_IN_ARGS(MACRO_WITH__PRAGMA) // expected-warning {{lower precedence}} \
+                                     // expected-note 2 {{place parentheses}} \
+                                     // expected-warning {{expression result unused}}
+// CHECK: #pragma warning(push)
+// CHECK: #pragma warning(disable: 10000)
+// CHECK: ; 1 + (2 > 3) ? 4 : 5;
+// CHECK: #pragma warning(pop)
+}
 
 // This should include macro_arg_directive even though the include
 // is looking for test.h  This allows us to assign to "n"
 #pragma include_alias("test.h", "macro_arg_directive.h" )
 #include "test.h"
 void test( void ) {
   n = 12;
 }