[Preprocessor] Implement empty vs omitted __VA_ARGS__ string-expansion MSVC Extension

[erichkeane]

Summary

The MSVC compiler extends the preprocessor by emitting an empty VA_ARGS expansion as NOTHING instead of "" (double-double quotes) in the case where the argument is omitted(rather than left empty. For example:
#define M(x,VA_ARGS)
M(5,) Argument is empty, thus this should result in ""
M(5) Argument is omitted, thus this should result in nothing.

[erichkeane]

This has likely gone stale, and my employer has decided to tell customers to modify their code.

[rnk]

Would this help with https://bugs.llvm.org/show_bug.cgi?id=32021 ?

[erichkeane]

I don't know? I don't have a really good way of repro'ing that. Do you have any ability to grab me a preprocessed reproducer of that one? Godbolt doesn't have Windows headers for this.

include/clang/Lex/MacroArgs.h

class MacroArgs {
   /// StringifiedArgs - This contains arguments in 'stringified' form.  If the
   /// stringified form of an argument has not yet been computed, this is empty.
   std::vector<Token> StringifiedArgs;
 
   /// ArgCache - This is a linked list of MacroArgs objects that the
   /// Preprocessor owns which we use to avoid thrashing malloc/free.
   MacroArgs *ArgCache;
 
+  /// ActualArgs - The number of arguments passed to this macro invocation.
+  unsigned NumActualArgs;
+
   /// MacroArgs - The number of arguments the invoked macro expects.
   unsigned NumMacroArgs;
 
-  MacroArgs(unsigned NumToks, bool varargsElided, unsigned MacroArgs)
+  MacroArgs(unsigned NumToks, bool varargsElided, unsigned ActualArgs,
+            unsigned MacroArgs)
       : NumUnexpArgTokens(NumToks), VarargsElided(varargsElided),
-        ArgCache(nullptr), NumMacroArgs(MacroArgs) {}
+        ArgCache(nullptr), NumActualArgs(ActualArgs), NumMacroArgs(MacroArgs) {}
   ~MacroArgs() = default;
 
 public:
   /// MacroArgs ctor function - Create a new MacroArgs object with the specified
   /// macro and argument info.
-  static MacroArgs *create(const MacroInfo *MI,
-                           ArrayRef<Token> UnexpArgTokens,
-                           bool VarargsElided, Preprocessor &PP);
+  static MacroArgs *create(const MacroInfo *MI, ArrayRef<Token> UnexpArgTokens,
+                           bool VarargsElided, unsigned ActualArgs,
+                           Preprocessor &PP);
 
   /// destroy - Destroy and deallocate the memory for this object.
   ///
   void destroy(Preprocessor &PP);
 
   /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
   /// by pre-expansion, return false.  Otherwise, conservatively return true.
   bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const;
const std::vector<Token> &
     getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP);
 
   /// getStringifiedArgument - Compute, cache, and return the specified argument
   /// that has been 'stringified' as required by the # operator.
   const Token &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP,
                                       SourceLocation ExpansionLocStart,
                                       SourceLocation ExpansionLocEnd);
 
+  /// getNumActualArguments - Return the number of arguments passed into this
+  /// macro invocation.
+  unsigned getNumActualArguments() const { return NumActualArgs; }
+
   /// getNumMacroArguments - Return the number of arguments the invoked macro
   /// expects.
   unsigned getNumMacroArguments() const { return NumMacroArgs; }
 
   /// isVarargsElidedUse - Return true if this is a C99 style varargs macro
   /// invocation and there was no argument specified for the "..." argument.  If
   /// the argument was specified (even empty) or this isn't a C99 style varargs
   /// function, or if in strict mode and the C99 varargs macro had only a ...
   /// argument, this returns false.
   bool isVarargsElidedUse() const { return VarargsElided; }
 
+  /// shouldOmitEmptyString - Return true if this is a MSVC Compatibility
+  /// situation, and this is a __VA_ARGS__ macro invocation where the variadic
+  /// argument was completely omitted.  This permits the MSVC Extension where
+  /// omitted stringified __VA_ARGS__ result in no expansion, but an empty
+  /// __VA_ARGS__ results in the "" string literal.
+  bool shouldOmitEmptyString(const Preprocessor &PP);
+
   /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
   /// tokens into the literal string token that should be produced by the C #
   /// preprocessor operator.  If Charify is true, then it should be turned into
-  /// a character literal for the Microsoft charize (#@) extension.
-  ///
-  static Token StringifyArgument(const Token *ArgToks,
-                                 Preprocessor &PP, bool Charify,
+  /// a character literal for the Microsoft charize (#@) extension. If
+  /// OmitEmptyString is true, this function will omit result in a blank token
+  /// rather than two quote characters.
+  static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP,
+                                 bool Charify, bool OmitEmptyString,
                                  SourceLocation ExpansionLocStart,
                                  SourceLocation ExpansionLocEnd);
-  
-  
   /// deallocate - This should only be called by the Preprocessor when managing
   /// its freelist.
   MacroArgs *deallocate();
 };
 
 }  // end namespace clang
 
 #endif

lib/Lex/MacroArgs.cpp

 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/SaveAndRestore.h"
 #include <algorithm>
 
 using namespace clang;
 
 /// MacroArgs ctor function - This destroys the vector passed in.
 MacroArgs *MacroArgs::create(const MacroInfo *MI,
-                             ArrayRef<Token> UnexpArgTokens,
-                             bool VarargsElided, Preprocessor &PP) {
+                             ArrayRef<Token> UnexpArgTokens, bool VarargsElided,
+                             unsigned ActualArgs, Preprocessor &PP) {
   assert(MI->isFunctionLike() &&
          "Can't have args for an object-like macro!");
   MacroArgs **ResultEnt = nullptr;
   unsigned ClosestMatch = ~0U;
   
   // See if we have an entry with a big enough argument list to reuse on the
   // free list.  If so, reuse it.
   for (MacroArgs **Entry = &PP.MacroArgCache; *Entry;
for (MacroArgs **Entry = &PP.MacroArgCache; *Entry;
     }
 
   MacroArgs *Result;
   if (!ResultEnt) {
     // Allocate memory for a MacroArgs object with the lexer tokens at the end.
     Result = (MacroArgs *)malloc(sizeof(MacroArgs) +
                                  UnexpArgTokens.size() * sizeof(Token));
     // Construct the MacroArgs object.
-    new (Result)
-        MacroArgs(UnexpArgTokens.size(), VarargsElided, MI->getNumArgs());
+    new (Result) MacroArgs(UnexpArgTokens.size(), VarargsElided, ActualArgs,
+                           MI->getNumArgs());
   } else {
     Result = *ResultEnt;
     // Unlink this node from the preprocessors singly linked list.
     *ResultEnt = Result->ArgCache;
     Result->NumUnexpArgTokens = UnexpArgTokens.size();
     Result->VarargsElided = VarargsElided;
+    Result->NumActualArgs = ActualArgs;
     Result->NumMacroArgs = MI->getNumArgs();
   }
 
   // Copy the actual unexpanded tokens to immediately after the result ptr.
   if (!UnexpArgTokens.empty())
     std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), 
               const_cast<Token*>(Result->getUnexpArgument(0)));
 
MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI,
   // that the token may be lexed sometime after the vector of tokens itself is
   // destroyed, which would be badness.
   if (PP.InCachingLexMode())
     PP.ExitCachingLexMode();
   PP.RemoveTopOfLexerStack();
   return Result;
 }
 
+/// shouldOmitEmptyString - Return true if this is a MSVC Compatibility
+/// situation, and this is a __VA_ARGS__ macro invocation where the variadic
+/// argument was completely omitted.  This permits the MSVC Extension where
+/// omitted stringified __VA_ARGS__ result in no expansion, but an empty
+/// __VA_ARGS__ results in the "" string literal.
+bool MacroArgs::shouldOmitEmptyString(const Preprocessor &PP) {
+  return PP.getLangOpts().MSVCCompat && isVarargsElidedUse() &&
+         getNumActualArguments() < getNumMacroArguments();
+}
 
 /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
 /// tokens into the literal string token that should be produced by the C #
 /// preprocessor operator.  If Charify is true, then it should be turned into
 /// a character literal for the Microsoft charize (#@) extension.
 ///
-Token MacroArgs::StringifyArgument(const Token *ArgToks,
-                                   Preprocessor &PP, bool Charify,
+Token MacroArgs::StringifyArgument(const Token *ArgToks, Preprocessor &PP,
+                                   bool Charify, bool OmitEmptyString,
                                    SourceLocation ExpansionLocStart,
                                    SourceLocation ExpansionLocEnd) {
   Token Tok;
   Tok.startToken();
   Tok.setKind(Charify ? tok::char_constant : tok::string_literal);
 
   const Token *ArgTokStart = ArgToks;
 
   // Stringify all the tokens.
   SmallString<128> Result;
+  // In the MSVC omitted __VA_ARGS__ case, and where there are no tokens
+  // immediately return an empty string token.
+  if (OmitEmptyString && ArgToks->is(tok::eof)) {
+    PP.CreateString(Result, Tok, ExpansionLocStart, ExpansionLocEnd);
+    return Tok;
+  }
+
   Result += "\"";
 
   bool isFirst = true;
   for (; ArgToks->isNot(tok::eof); ++ArgToks) {
     const Token &Tok = *ArgToks;
     if (!isFirst && (Tok.hasLeadingSpace() || Tok.isAtStartOfLine()))
       Result += ' ';
     isFirst = false;
const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
                                                Preprocessor &PP,
                                                SourceLocation ExpansionLocStart,
                                                SourceLocation ExpansionLocEnd) {
   assert(ArgNo < getNumMacroArguments() && "Invalid argument number!");
   if (StringifiedArgs.empty())
     StringifiedArgs.resize(getNumMacroArguments(), {});
 
   if (StringifiedArgs[ArgNo].isNot(tok::string_literal))
-    StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP,
-                                               /*Charify=*/false,
-                                               ExpansionLocStart,
-                                               ExpansionLocEnd);
+    StringifiedArgs[ArgNo] =
+        StringifyArgument(getUnexpArgument(ArgNo), PP,
+                          /*Charify=*/false, shouldOmitEmptyString(PP),
+                          ExpansionLocStart, ExpansionLocEnd);
   return StringifiedArgs[ArgNo];
 }

lib/Lex/PPMacroExpansion.cpp

if (NumActuals < MinArgsExpected) {
     // Emit the diagnostic at the macro name in case there is a missing ).
     // Emitting it at the , could be far away from the macro name.
     Diag(MacroName, diag::err_too_many_args_in_macro_invoc);
     Diag(MI->getDefinitionLoc(), diag::note_macro_here)
       << MacroName.getIdentifierInfo();
     return nullptr;
   }
 
-  return MacroArgs::create(MI, ArgTokens, isVarargsElided, *this);
+  return MacroArgs::create(MI, ArgTokens, isVarargsElided, NumActuals, *this);
 }
 
 /// \brief Keeps macro expanded tokens for TokenLexers.
 //
 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
 /// going to lex in the cache and when it finishes the tokens are removed
 /// from the end of the cache.
 Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer,

lib/Lex/TokenLexer.cpp

if (CurTok.isOneOf(tok::hash, tok::hashat)) {
 
       Token Res;
       if (CurTok.is(tok::hash))  // Stringify
         Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
                                                  ExpansionLocStart,
                                                  ExpansionLocEnd);
       else {
         // 'charify': don't bother caching these.
-        Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
-                                           PP, true,
-                                           ExpansionLocStart,
-                                           ExpansionLocEnd);
+        Res = MacroArgs::StringifyArgument(
+            ActualArgs->getUnexpArgument(ArgNo), PP, true,
+            /*OmitEmptyString=*/false, ExpansionLocStart, ExpansionLocEnd);
       }
       Res.setFlag(Token::StringifiedInMacro);
 
       // The stringified/charified string leading space flag gets set to match
       // the #/#@ operator.
       if (NextTokGetsSpace)
         Res.setFlag(Token::LeadingSpace);
 

test/Preprocessor/microsoft-ext.c

 // RUN: %clang_cc1 -E -fms-compatibility %s -o %t
 // RUN: FileCheck %s < %t
 
 # define M2(x, y) x + y
 # define P(x, y) {x, y}
 # define M(x, y) M2(x, P(x, y))
 M(a, b) // CHECK: a + {a, b}
+#undef M
+#undef P
+#undef M2
 
 // Regression test for PR13924
 #define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar)
 #define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar
 
 #define GMOCK_INTERNAL_COUNT_AND_2_VALUE_PARAMS(p0, p1) P2
 
 #define GMOCK_ACTION_CLASS_(name, value_params)\

 
 #define MAKER(_arg1, _comma, _arg2)                                            \
   void func(_arg1 _comma _arg2) {}
 #define MAKE_FUNC(_makerP1, _makerP2, _arg1, _comma, _arg2)                    \
   _makerP1##_makerP2(_arg1, _comma, _arg2)
 
 MAKE_FUNC(MAK, ER, int a, _COMMA, int b);
 // CHECK: void func(int a , int b) {}
+#undef MAKE_FUNC
+#undef MAKER
+#undef _COMMA
 
 #define macro(a, b) (a - b)
 void function(int a);
 #define COMMA_ELIDER(...) \
   macro(x, __VA_ARGS__); \
   function(x, __VA_ARGS__);
 COMMA_ELIDER();
 // CHECK: (x - );
 // CHECK: function(x);
+#undef COMMA_ELIDER
+#undef macro
+
+// When expanding stringized __VA_ARGS__, MSVC expands omitted __VA_ARGS__ as nothing,
+// and empty __VA_ARGS__ as the empty-string.
+#define M(x, ...) #__VA_ARGS__
+Many is M(a,b,c)BAR
+// CHECK: Many is "b,c"BAR
+Empty is M(a,)BAR
+// CHECK: Empty is ""BAR
+Omitted is M(a)BAR
+// CHECK: Omitted is BAR
+#undef M
+