[Preprocessor] Implement MSVC extension to expand arguments to ##

[erichkeane]

Summary

MSVC expands arguments to "##" anyway, so this implements it as an extension. In order to accomplish this, expansions of single fucntion arguments were pulled into a separate function.

[erichkeane]

This has likely gone stale, and my employer has decided to tell customers to modify their code.

include/clang/Lex/TokenLexer.h

private:
   /// Tok.  If this returns true, the caller should immediately return the
   /// token.
   bool PasteTokens(Token &Tok);
 
   /// Expand the arguments of a function-like macro so that we can quickly
   /// return preexpanded tokens from Tokens.
   void ExpandFunctionArguments();
 
+  /// Expand a single argument of a function-like macro, so that
+  /// ExpandFunctionArguments can use it to quickly return preexpanded tokens
+  /// from Tokens.  Returns true if a non-empty token was expanded.
+  bool ExpandSingleFunctionArgument(const Token &CurTok, int ArgNo,
+                                    SmallVector<Token, 128> &ResultToks);
+
   /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
   /// together to form a comment that comments out everything in the current
   /// macro, other active macros, and anything left on the current physical
   /// source line of the expanded buffer.  Handle this by returning the
   /// first token on the next line.
   void HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc);
 
   /// \brief If \p loc is a FileID and points inside the current macro

lib/Lex/TokenLexer.cpp

if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
                                      /*HasPasteOperator=*/false,
                                      Macro, ArgNo, PP))
       continue;
 
     // If it is not the LHS/RHS of a ## operator, we must pre-expand the
     // argument and substitute the expanded tokens into the result.  This is
     // C99 6.10.3.1p1.
     if (!PasteBefore && !PasteAfter) {
-      const Token *ResultArgToks;
-
-      // Only preexpand the argument if it could possibly need it.  This
-      // avoids some work in common cases.
-      const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
-      if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
-        ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0];
-      else
-        ResultArgToks = ArgTok;  // Use non-preexpanded tokens.
-
-      // If the arg token expanded into anything, append it.
-      if (ResultArgToks->isNot(tok::eof)) {
-        size_t FirstResult = ResultToks.size();
-        unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
-        ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
-
-        // In Microsoft-compatibility mode, we follow MSVC's preprocessing
-        // behavior by not considering single commas from nested macro
-        // expansions as argument separators. Set a flag on the token so we can
-        // test for this later when the macro expansion is processed.
-        if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
-            ResultToks.back().is(tok::comma))
-          ResultToks.back().setFlag(Token::IgnoredComma);
-
-        // If the '##' came from expanding an argument, turn it into 'unknown'
-        // to avoid pasting.
-        for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
-                                           ResultToks.end())) {
-          if (Tok.is(tok::hashhash))
-            Tok.setKind(tok::unknown);
-        }
-
-        if(ExpandLocStart.isValid()) {
-          updateLocForMacroArgTokens(CurTok.getLocation(),
-                                     ResultToks.begin()+FirstResult,
-                                     ResultToks.end());
-        }
-
-        // If any tokens were substituted from the argument, the whitespace
-        // before the first token should match the whitespace of the arg
-        // identifier.
-        ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
-                                             NextTokGetsSpace);
-        ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
-        NextTokGetsSpace = false;
-      }
+      ExpandSingleFunctionArgument(CurTok, ArgNo, ResultToks);
       continue;
     }
 
     // Okay, we have a token that is either the LHS or RHS of a paste (##)
     // argument.  It gets substituted as its non-pre-expanded tokens.
     const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
     unsigned NumToks = MacroArgs::getArgLength(ArgToks);
     if (NumToks) {  // Not an empty argument?
       // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
       // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
       // the expander trys to paste ',' with the first token of the __VA_ARGS__
       // expansion.
       if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
           ResultToks[ResultToks.size()-2].is(tok::comma) &&
           (unsigned)ArgNo == Macro->getNumArgs()-1 &&
           Macro->isVariadic()) {
         // Remove the paste operator, report use of the extension.
         PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
       }
 
-      ResultToks.append(ArgToks, ArgToks+NumToks);
+      // MSVC Expands arguments to ## anyway.
+      if (PP.getLangOpts().MSVCCompat) {
+        if (!ExpandSingleFunctionArgument(CurTok, ArgNo, ResultToks)) {
+          // If the token doesn't exist, replace it with an empty string so that
+          // token pasting will work.
+          Token BlankTok;
+          BlankTok.startToken();
+          BlankTok.setKind(tok::string_literal);
+          PP.CreateString("", BlankTok);
+
+          if (!ResultToks.empty()) {
+            ResultToks.back().setFlagValue(Token::LeadingSpace,
+                                           NextTokGetsSpace);
+            ResultToks.back().setFlagValue(Token::StartOfLine, false);
+            NextTokGetsSpace = false;
+          }
+
+          ResultToks.push_back(BlankTok);
+          if (ExpandLocStart.isValid())
+            updateLocForMacroArgTokens(CurTok.getLocation(),
+                                       ResultToks.end() - 1, ResultToks.end());
+        }
+      } else {
+        ResultToks.append(ArgToks, ArgToks + NumToks);
 
         // If the '##' came from expanding an argument, turn it into 'unknown'
         // to avoid pasting.
-      for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
-                                         ResultToks.end())) {
+        for (Token &Tok :
+             llvm::make_range(ResultToks.end() - NumToks, ResultToks.end())) {
           if (Tok.is(tok::hashhash))
             Tok.setKind(tok::unknown);
         }
 
         if (ExpandLocStart.isValid()) {
           updateLocForMacroArgTokens(CurTok.getLocation(),
-                                   ResultToks.end()-NumToks, ResultToks.end());
+                                     ResultToks.end() - NumToks,
+                                     ResultToks.end());
+        }
       }
 
       // If this token (the macro argument) was supposed to get leading
       // whitespace, transfer this information onto the first token of the
       // expansion.
       //
       // Do not do this if the paste operator occurs before the macro argument,
       // as in "A ## MACROARG".  In valid code, the first token will get
if (MadeChange) {
     // when this TokenLexer finishes lexing them.
     Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
 
     // The preprocessor cache of macro expanded tokens owns these tokens,not us.
     OwnsTokens = false;
   }
 }
 
+/// Expand a single argument of a function-like macro, so that
+/// ExpandFunctionArguments can use it to quickly return preexpanded tokens
+/// from Tokens.  Returns true if the argument expanded to anything.
+bool TokenLexer::ExpandSingleFunctionArgument(
+    const Token &CurTok, int ArgNo, SmallVector<Token, 128> &ResultToks) {
+  const Token *UnexpArgs = ActualArgs->getUnexpArgument(ArgNo);
+  const Token *ExpArgs = nullptr;
+
+  // Only preexpand the argument if it could possibly need it.  This
+  // avoids some work in common cases.
+  if (ActualArgs->ArgNeedsPreexpansion(UnexpArgs, PP))
+    ExpArgs = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0];
+  else
+    ExpArgs = UnexpArgs; // Use non-preexpanded tokens.
+
+  // If the arg token expanded into anything, append it.
+  if (ExpArgs->isNot(tok::eof)) {
+    size_t FirstResult = ResultToks.size();
+    unsigned NumToks = MacroArgs::getArgLength(ExpArgs);
+    ResultToks.append(ExpArgs, ExpArgs + NumToks);
+
+    // In Microsoft-compatibility mode, we follow MSVC's preprocessing
+    // behavior by not considering single commas from nested macro
+    // expansions as argument separators. Set a flag on the token so we can
+    // test for this later when the macro expansion is processed.
+    if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
+        ResultToks.back().is(tok::comma))
+      ResultToks.back().setFlag(Token::IgnoredComma);
+
+    // If the '##' came from expanding an argument, turn it into 'unknown'
+    // to avoid pasting.
+    for (Token &Tok :
+         llvm::make_range(ResultToks.begin() + FirstResult, ResultToks.end())) {
+      if (Tok.is(tok::hashhash))
+        Tok.setKind(tok::unknown);
+    }
+
+    if (ExpandLocStart.isValid()) {
+      updateLocForMacroArgTokens(CurTok.getLocation(),
+                                 ResultToks.begin() + FirstResult,
+                                 ResultToks.end());
+    }
+
+    // If any tokens were substituted from the argument, the whitespace
+    // before the first token should match the whitespace of the arg
+    // identifier.
+    ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, NextTokGetsSpace);
+    ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
+    NextTokGetsSpace = false;
+    return true;
+  }
+  return false;
+}
+
 /// \brief Checks if two tokens form wide string literal.
 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
                                          const Token &SecondTok) {
   return FirstTok.is(tok::identifier) &&
          FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
          SecondTok.stringifiedInMacro();
 }
 

test/Preprocessor/microsoft-ext.c

#define COMMA_ELIDER(...) \
   macro(x, __VA_ARGS__); \
   function(x, __VA_ARGS__);
 COMMA_ELIDER();
 // CHECK: (x - );
 // CHECK: function(x);
 #undef COMMA_ELIDER
 #undef macro
 
-// When expanding stringized __VA_ARGS__, MSVC expands omitted __VA_ARGS__ as nothing,
-// and empty __VA_ARGS__ as the empty-string.
+// When expanding stringized __VA_ARGS__, MSVC expands omitted __VA_ARGS__ as 
+// nothing and empty __VA_ARGS__ as the empty-string.
 #define M(x, ...) #__VA_ARGS__
 Many is M(a,b,c)BAR
 // CHECK: Many is "b,c"BAR
 Empty is M(a,)BAR
 // CHECK: Empty is ""BAR
 Omitted is M(a)BAR
 // CHECK: Omitted is BAR
 #undef M
 
+// MSVC Preprocessor expands arguments to the ## operator.
+#define M(a, ...) a ## __VA_ARGS__
+#define TWO 2
+TWELVE: M(1,TWO)
+// CHECK: TWELVE: 12
+TWELVE_A: M(1, TWO)
+// CHECK: TWELVE_A: 12
+TWENTYONE: M(TWO,1)
+// CHECK: TWENTYONE: 21
+TWENTYONE_A: M(TWO, 1)
+// CHECK: TWENTYONE_A: 21
+#undef TWO
+#undef M
+
+// Also validate the situation where "TWO" is empty.
+#define M(a, ...) a ## __VA_ARGS__
+#define TWO
+TWELVE_EMPTY:M(1,TWO)
+// CHECK: TWELVE_EMPTY:1
+TWELVE_EMPTY_A: M(1,TWO)
+// CHECK: TWELVE_EMPTY_A: 1
+TWENTYONE_EMPTY:M(TWO, 1)
+// CHECK: TWENTYONE_EMPTY:1
+TWENTYONE_EMPTY_A: M(TWO, 1)
+// CHECK: TWENTYONE_EMPTY_A: 1
+#undef TWO
+#undef M