[ms] [llvm-ml] Match ML's handling of macro parameter expansion

[epastor]

Summary

Matching ML.EXE, we expand tokens in macro parameters only if the parameter starts with a % character.

Also matching ML.EXE's behavior: If a macro parameter does start with a % character, unless the parameter consists of a single non-numeric macro, we also try to evaluate the resulting expanded parameter as an absolute expression if possible. If this succeeds, the parameter is replaced by its numeric value expressed in decimal. The exceptional case mentioned allows for parameters such as %@Date (where @Date renders in MM/DD/YY format) without accidental evaluation as a numeric expression.

[epastor]

Rebase on parent

llvm/lib/MC/MCParser/MasmParser.cpp

public:
   void Note(SMLoc L, const Twine &Msg, SMRange Range = None) override;
   bool Warning(SMLoc L, const Twine &Msg, SMRange Range = None) override;
   bool printError(SMLoc L, const Twine &Msg, SMRange Range = None) override;
 
   enum ExpandKind { ExpandMacros, DoNotExpandMacros };
   const AsmToken &Lex(ExpandKind ExpandNextToken);
   const AsmToken &Lex() override { return Lex(ExpandMacros); }
 
+  using MCAsmParser::parseToken;
+  bool parseToken(AsmToken::TokenKind T, const Twine &Msg,
+                  ExpandKind ExpandNextToken);
+
   void setParsingMSInlineAsm(bool V) override {
     ParsingMSInlineAsm = V;
     // When parsing MS inline asm, we must lex 0b1101 and 0ABCH as binary and
     // hex integer literals.
     Lexer.setLexMasmIntegers(V);
   }
   bool isParsingMSInlineAsm() override { return ParsingMSInlineAsm; }
 
private:
 
   /// Parse all macro arguments for a given macro.
   bool
   parseMacroArguments(const MCAsmMacro *M, MCAsmMacroArguments &A,
                       AsmToken::TokenKind EndTok = AsmToken::EndOfStatement);
 
   void printMacroInstantiations();
 
+  bool expandString(std::string Body, SMLoc StartLoc, SMLoc EndLoc);
   bool expandStatement(SMLoc Loc);
 
   void printMessage(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Msg,
                     SMRange Range = None) const {
     ArrayRef<SMRange> Ranges(Range);
     SrcMgr.PrintMessage(Loc, Kind, Msg, Ranges);
   }
   static void DiagHandler(const SMDiagnostic &Diag, void *Context);
CurBuffer =
       SrcMgr.AddNewSourceBuffer(std::move(Instantiation), Tok.getEndLoc());
   Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer(), nullptr,
                   /*EndStatementAtEOF=*/false);
   EndStatementAtEOFStack.push_back(false);
   Lexer.Lex();
   return false;
 }
 
+bool MasmParser::parseToken(AsmToken::TokenKind T, const Twine &Msg,
+                            ExpandKind ExpandNextToken) {
+  if (getTok().getKind() != T)
+    return Error(getTok().getLoc(), Msg);
+  Lex(ExpandNextToken);
+  return false;
+}
+
 const AsmToken &MasmParser::Lex(ExpandKind ExpandNextToken) {
   if (Lexer.getTok().is(AsmToken::Error))
     Error(Lexer.getErrLoc(), Lexer.getErr());
 
   // if it's a end of statement with a comment in it
   if (getTok().is(AsmToken::EndOfStatement)) {
     // if this is a line comment output it.
     if (!getTok().getString().empty() && getTok().getString().front() != '\n' &&
if (Lexer.is(AsmToken::Eof) || Lexer.is(AsmToken::Equal))
       return TokError("unexpected token");
 
     if (ParenLevel == 0) {
       if (Lexer.is(AsmToken::Comma))
         break;
 
       if (Lexer.is(AsmToken::Space)) {
         SpaceEaten = true;
-        Lex(); // Eat spaces.
+        Lex(DoNotExpandMacros); // Eat spaces.
       }
 
       // Spaces can delimit parameters, but could also be part an expression.
       // If the token after a space is an operator, add the token and the next
       // one into this argument
       if (!IsDarwin) {
         if (isOperator(Lexer.getKind()) && Lexer.isNot(EndTok)) {
           MA.push_back(getTok());
-          Lex();
+          Lex(DoNotExpandMacros);
 
           // Whitespace after an operator can be ignored.
           if (Lexer.is(AsmToken::Space))
-            Lex();
+            Lex(DoNotExpandMacros);
 
           continue;
         }
       }
       if (SpaceEaten)
         break;
     }
 
     // handleMacroEntry relies on not advancing the lexer here
     // to be able to fill in the remaining default parameter values
     if (Lexer.is(EndTok) && (EndTok != AsmToken::RParen || ParenLevel == 0))
       break;
 
     // Adjust the current parentheses level.
     if (Lexer.is(AsmToken::LParen))
       ++ParenLevel;
     else if (Lexer.is(AsmToken::RParen) && ParenLevel)
       --ParenLevel;
 
     // Append the token to the current argument list.
     MA.push_back(getTok());
-    Lex();
+    Lex(DoNotExpandMacros);
   }
 
   if (ParenLevel != 0)
     return TokError("unbalanced parentheses in argument");
 
   if (MA.empty() && MP) {
     if (MP->Required) {
       return TokError("missing value for required parameter '" + MP->Name +
for (unsigned Parameter = 0; !NParameters || Parameter < NParameters;
 
     if (Lexer.is(AsmToken::Identifier) && peekTok().is(AsmToken::Equal)) {
       if (parseIdentifier(FA.Name))
         return Error(IDLoc, "invalid argument identifier for formal argument");
 
       if (Lexer.isNot(AsmToken::Equal))
         return TokError("expected '=' after formal parameter identifier");
 
-      Lex();
+      Lex(DoNotExpandMacros);
 
       NamedParametersFound = true;
     }
 
     if (NamedParametersFound && FA.Name.empty())
       return Error(IDLoc, "cannot mix positional and keyword arguments");
 
     unsigned PI = Parameter;
if (!FA.Name.empty()) {
       }
       PI = FAI;
     }
     const MCAsmMacroParameter *MP = nullptr;
     if (M && PI < NParameters)
       MP = &M->Parameters[PI];
 
     SMLoc StrLoc = Lexer.getLoc();
-    SMLoc EndLoc;
+    bool Expand = false;
     if (Lexer.is(AsmToken::Percent)) {
-      const MCExpr *AbsoluteExp;
-      int64_t Value;
-      /// Eat '%'.
-      Lex();
-      if (parseExpression(AbsoluteExp, EndLoc))
-        return false;
-      if (!AbsoluteExp->evaluateAsAbsolute(Value,
-                                           getStreamer().getAssemblerPtr()))
-        return Error(StrLoc, "expected absolute expression");
-      const char *StrChar = StrLoc.getPointer();
-      const char *EndChar = EndLoc.getPointer();
-      AsmToken newToken(AsmToken::Integer,
-                        StringRef(StrChar, EndChar - StrChar), Value);
-      FA.Value.push_back(newToken);
-    } else if (parseMacroArgument(MP, FA.Value, EndTok)) {
+      // Eat initial '%', and then expand the argument.
+      Lex(DoNotExpandMacros);
+      Expand = true;
+    }
+    if (parseMacroArgument(MP, FA.Value, EndTok)) {
       if (M)
         return addErrorSuffix(" in '" + M->Name + "' macro");
       else
         return true;
     }
+    if (Expand) {
+      // Unless the argument was a single text macro, we also want to try to
+      // evaluate the expanded result as an absolute expression.
+      bool Evaluate = true;
+      if (FA.Value.size() == 1 && FA.Value.front().is(AsmToken::Identifier)) {
+        std::string IDLower = FA.Value.front().getIdentifier().lower();
+        auto BuiltinIt = BuiltinSymbolMap.find(IDLower);
+        auto VarIt = Variables.find(IDLower);
+        if (BuiltinIt != BuiltinSymbolMap.end()) {
+          Evaluate =
+              (evaluateBuiltinValue(BuiltinIt->getValue(), StrLoc) != nullptr);
+        } else if (VarIt != Variables.end()) {
+          Evaluate = !VarIt->getValue().IsText;
+        }
+      }
+
+      // Copy the argument into a new buffer to expand it.
+      SmallString<256> Buf;
+      raw_svector_ostream OS(Buf);
+      for (const AsmToken &Token : FA.Value) {
+        OS << Token.getString();
+      }
+      SMLoc StrEndLoc = getTok().getLoc();
+      unsigned OriginalBuffer = CurBuffer;
+      if (expandString(OS.str().str(), StrLoc, StrEndLoc))
+        return true;
+      unsigned ExpandedBuffer = CurBuffer;
+      StringRef ExpandedArgument =
+          SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
+      FA.Value.clear();
+      FA.Value.push_back(AsmToken(AsmToken::Identifier, ExpandedArgument));
+
+      // Unless the argument was a single text macro, try to parse the expanded
+      // version as an absolute expression; if successful, replace it by its
+      // evaluation.
+      printPendingErrors();
+      const MCExpr *AbsoluteExp;
+      int64_t Value;
+      SMLoc EndLoc;
+      if (Evaluate && !parseExpression(AbsoluteExp, EndLoc) &&
+          AbsoluteExp->evaluateAsAbsolute(Value,
+                                          getStreamer().getAssemblerPtr()) &&
+          CurBuffer == ExpandedBuffer && Lexer.is(AsmToken::EndOfStatement) &&
+          Lexer.Lex().is(AsmToken::Eof)) {
+        const char *StrChar = StrLoc.getPointer();
+        const char *EndChar = EndLoc.getPointer();
+        FA.Value.clear();
+        FA.Value.push_back(AsmToken(
+            AsmToken::Integer, StringRef(StrChar, EndChar - StrChar), Value));
+      }
+
+      // Disregard any errors that occurred while trying to parse the
+      // argument as an expression; resume parsing at the original location.
+      clearPendingErrors();
+      EndStatementAtEOFStack.pop_back();
+      jumpToLoc(StrEndLoc, OriginalBuffer, EndStatementAtEOFStack.back());
+      Lex(DoNotExpandMacros);
+    }
 
     if (!FA.Value.empty()) {
       if (A.size() <= PI)
         A.resize(PI + 1);
       A[PI] = FA.Value;
 
       if (FALocs.size() <= PI)
         FALocs.resize(PI + 1);
if (Lexer.is(EndTok)) {
           if (!M->Parameters[FAI].Value.empty())
             A[FAI] = M->Parameters[FAI].Value;
         }
       }
       return Failure;
     }
 
     if (Lexer.is(AsmToken::Comma))
-      Lex();
+      Lex(DoNotExpandMacros);
   }
 
   return TokError("too many positional arguments");
 }
 
 bool MasmParser::handleMacroEntry(const MCAsmMacro *M, SMLoc NameLoc,
                                   AsmToken::TokenKind ArgumentEndTok) {
   // Arbitrarily limit macro nesting depth (default matches 'as'). We can
if (nextTok.isNot(AsmToken::Identifier))
       return true;
 
     // We have a '$' or '@' followed by an identifier, make sure they are adjacent.
     if (PrefixLoc.getPointer() + 1 != nextTok.getLoc().getPointer())
       return true;
 
     // eat $ or @
     Lexer.Lex(); // Lexer's Lex guarantees consecutive token.
-    // Construct the joined identifier and consume the token.
+    // Construct the joined identifier.
     Res =
         StringRef(PrefixLoc.getPointer(), getTok().getIdentifier().size() + 1);
-    Lex(); // Parser Lex to maintain invariants.
-    return false;
-  }
-
+  } else {
     if (Lexer.isNot(AsmToken::Identifier) && Lexer.isNot(AsmToken::String))
       return true;
-
     Res = getTok().getIdentifier();
+  }
 
-  // Consume the identifier token - but if parsing certain directives, avoid
-  // lexical expansion of the next token.
+  // Consume the identifier token - but if parsing certain directives or a macro
+  // invocation, avoid lexical expansion of the next token.
   ExpandKind ExpandNextToken = ExpandMacros;
-  if (Position == StartOfStatement &&
-      StringSwitch<bool>(Res)
-          .CaseLower("echo", true)
-          .CasesLower("ifdef", "ifndef", "elseifdef", "elseifndef", true)
-          .Default(false)) {
+  if (StartOfStatement) {
+    std::string IDLower = Res.lower();
+    if (IDLower == "echo" || IDLower == "ifdef" || IDLower == "ifndef" ||
+        IDLower == "elseifdef" || IDLower == "elseifndef") {
       ExpandNextToken = DoNotExpandMacros;
+    } else if (getContext().lookupMacro(Res.lower())) {
+      ExpandNextToken = DoNotExpandMacros;
+    }
   }
   Lex(ExpandNextToken);
 
   return false;
 }
 
 /// parseDirectiveEquate:
 ///  ::= name "=" expression
MCAsmMacro *MasmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
   const char *BodyEnd = EndToken.getLoc().getPointer();
   StringRef Body = StringRef(BodyStart, BodyEnd - BodyStart);
 
   // We Are Anonymous.
   MacroLikeBodies.emplace_back(StringRef(), Body, MCAsmMacroParameters());
   return &MacroLikeBodies.back();
 }
 
-bool MasmParser::expandStatement(SMLoc Loc) {
-  std::string Body = parseStringTo(AsmToken::EndOfStatement);
-  SMLoc EndLoc = getTok().getLoc();
-
+bool MasmParser::expandString(std::string Body, SMLoc StartLoc, SMLoc EndLoc) {
   MCAsmMacroParameters Parameters;
   MCAsmMacroArguments Arguments;
 
-  StringMap<std::string> BuiltinValues;
+  StringMap<std::string> ExpandedValues;
   for (const auto &S : BuiltinSymbolMap) {
     const BuiltinSymbol &Sym = S.getValue();
-    if (llvm::Optional<std::string> Text = evaluateBuiltinTextMacro(Sym, Loc)) {
-      BuiltinValues[S.getKey().lower()] = std::move(*Text);
+    if (llvm::Optional<std::string> Text =
+            evaluateBuiltinTextMacro(Sym, StartLoc)) {
+      ExpandedValues[S.getKey().lower()] = std::move(*Text);
     }
   }
-  for (const auto &B : BuiltinValues) {
-    MCAsmMacroParameter P;
-    MCAsmMacroArgument A;
-    P.Name = B.getKey();
-    P.Required = true;
-    A.push_back(AsmToken(AsmToken::String, B.getValue()));
-
-    Parameters.push_back(std::move(P));
-    Arguments.push_back(std::move(A));
-  }
-
   for (const auto &V : Variables) {
     const Variable &Var = V.getValue();
     if (Var.IsText) {
-      MCAsmMacroParameter P;
-      MCAsmMacroArgument A;
-      P.Name = Var.Name;
-      P.Required = true;
-      A.push_back(AsmToken(AsmToken::String, Var.TextValue));
+      ExpandedValues[Var.Name] = Var.TextValue;
+    }
+  }
+  for (const auto &Expanded : ExpandedValues) {
+    P.Name = Expanded.getKey();
+    A.push_back(AsmToken(AsmToken::String, Expanded.getValue()));
 
-      Parameters.push_back(std::move(P));
-      Arguments.push_back(std::move(A));
-    }
-  }
+
   MacroLikeBodies.emplace_back(StringRef(), Body, Parameters);
   MCAsmMacro M = MacroLikeBodies.back();
 
   // Expand the statement in a new buffer.
   SmallString<80> Buf;
   raw_svector_ostream OS(Buf);
   if (expandMacro(OS, M.Body, M.Parameters, Arguments, M.Locals, EndLoc))
     return true;
   std::unique_ptr<MemoryBuffer> Expansion =
       MemoryBuffer::getMemBufferCopy(OS.str(), "<expansion>");
 
   // Jump to the expanded statement and prime the lexer.
   CurBuffer = SrcMgr.AddNewSourceBuffer(std::move(Expansion), EndLoc);
   Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer());
   EndStatementAtEOFStack.push_back(false);
   Lex();
   return false;
 }
 
+bool MasmParser::expandStatement(SMLoc Loc) {
+  std::string Body = parseStringTo(AsmToken::EndOfStatement);
+  SMLoc EndLoc = getTok().getLoc();
+  return expandString(Body, /*StartLoc=*/Loc, EndLoc);
+}
+
 void MasmParser::instantiateMacroLikeBody(MCAsmMacro *M, SMLoc DirectiveLoc,
                                           raw_svector_ostream &OS) {
   instantiateMacroLikeBody(M, DirectiveLoc, /*ExitLoc=*/getTok().getLoc(), OS);
 }
 void MasmParser::instantiateMacroLikeBody(MCAsmMacro *M, SMLoc DirectiveLoc,
                                           SMLoc ExitLoc,
                                           raw_svector_ostream &OS) {
   OS << "endm\n";

llvm/test/tools/llvm-ml/macro.asm

 ; CHECK-LABEL: purge_test:
 
   LocalSymbolMacro
 ; CHECK: "??0002":
 ; CHECK-NEXT: xor eax, eax
 ; CHECK-NEXT: jmp "??0002"
 purge_test ENDP
 
+SubstitutionInStringMacro MACRO x
+  BYTE "&x";
+ENDM
+
+substitution_in_string_test PROC
+
+  Var TEXTEQU <2+3>
+
+does_not_evaluate:
+; should expand to BYTE "2+3"
+  SubstitutionInStringMacro %Var
+; CHECK-LABEL: does_not_evaluate:
+; CHECK-NEXT: .byte 50
+; CHECK-NEXT: .byte 43
+; CHECK-NEXT: .byte 51
+; CHECK-NOT: .byte
+
+does_evaluate_left:
+; should expand to BYTE "9"
+  SubstitutionInStringMacro %Var+4
+; CHECK-LABEL: does_evaluate_left:
+; CHECK-NEXT: .byte 57
+; CHECK-NOT: .byte
+
+does_evaluate_right:
+; should expand to BYTE "9"
+  SubstitutionInStringMacro %4+Var
+; CHECK-LABEL: does_evaluate_right:
+; CHECK-NEXT: .byte 57
+; CHECK-NOT: .byte
+
+  Var TEXTEQU <t>
+
+expands:
+; should expand to BYTE "t"
+  SubstitutionInStringMacro %Var
+; CHECK-LABEL: expands:
+; CHECK-NEXT: .byte 116
+; CHECK-NOT: .byte
+
+does_not_evaluate_left:
+; should expand to BYTE "t+4"
+  SubstitutionInStringMacro %Var+4
+; CHECK-LABEL: does_not_evaluate_left:
+; CHECK-NEXT: .byte 116
+; CHECK-NEXT: .byte 43
+; CHECK-NEXT: .byte 52
+; CHECK-NOT: .byte
+
+does_not_evaluate_right:
+; should expand to BYTE "4+t"
+  SubstitutionInStringMacro %4+Var
+; CHECK-LABEL: does_not_evaluate_right:
+; CHECK-NEXT: .byte 52
+; CHECK-NEXT: .byte 43
+; CHECK-NEXT: .byte 116
+; CHECK-NOT: .byte
+
+; Mimic @Date format
+  Var TEXTEQU <06/03/02>
+
+date_expands:
+; should expand to BYTE "06/03/02"
+  SubstitutionInStringMacro %Var
+; CHECK-LABEL: date_expands:
+; CHECK-NEXT: .byte 48
+; CHECK-NEXT: .byte 54
+; CHECK-NEXT: .byte 47
+; CHECK-NEXT: .byte 48
+; CHECK-NEXT: .byte 51
+; CHECK-NEXT: .byte 47
+; CHECK-NEXT: .byte 48
+; CHECK-NEXT: .byte 50
+; CHECK-NOT: .byte
+
+pseudo_date_evaluates:
+; should expand to BYTE "3"
+  SubstitutionInStringMacro %Var+2
+; CHECK-LABEL: pseudo_date_evaluates:
+; CHECK-NEXT: .byte 51
+; CHECK-NOT: .byte
+
+substitution_in_string_test ENDP
+
+
 END