[ms] [llvm-ml] Lex MASM strings, including escaping

[epastor]

Summary

Allow single-quoted strings and double-quoted character values, as well as doubled-quote escaping.

[thakis]

This is missing tests for almost all of the functionality added in this patch ('asdf''''jkl' -> asdf''jkl but "asdf''''jkl" -> asdf''''jkl, similar for with ' and " swapped, and the single-char case).

llvm/lib/MC/MCParser/AsmLexer.cpp

608

put a return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart)); here, and leave the rest of the function unmodified

[epastor]

Reduce diff, and add specific tests of new functionality

[epastor]

Add test for handling of single characters in expressions (even in double-quotes)

[epastor]

Add tests for quote characters in expressions, with and without escaping

[epastor]

Tests added; thank you!

[thakis]

lg if the lack of \0-termination is intentional

llvm/test/tools/llvm-ml/strings.test

16

just to verify, these strings are not 0-terminated? (why is the single-char case a special case then?)

97

Oh I guess for this.

[epastor]

llvm/test/tools/llvm-ml/strings.test

16

Correct - null termination has to be manually added by passing a zero initializer after the string. (e.g., https://stackoverflow.com/q/19033872)

llvm/include/llvm/MC/MCParser/AsmLexer.h

size_t peekTokens(MutableArrayRef<AsmToken> Buf,
                     bool ShouldSkipSpace = true) override;
 
   const MCAsmInfo &getMAI() const { return MAI; }
 
 private:
   bool isAtStartOfComment(const char *Ptr);
   bool isAtStatementSeparator(const char *Ptr);
   int getNextChar();
+  int peekNextChar();
   AsmToken ReturnError(const char *Loc, const std::string &Msg);
 
   AsmToken LexIdentifier();
   AsmToken LexSlash();
   AsmToken LexLineComment();
   AsmToken LexDigit();
   AsmToken LexSingleQuote();
   AsmToken LexQuote();

llvm/include/llvm/MC/MCParser/MCAsmLexer.h

 
 protected: // Can only create subclasses.
   const char *TokStart = nullptr;
   bool SkipSpace = true;
   bool AllowAtInIdentifier;
   bool IsAtStartOfStatement = true;
   bool LexMasmHexFloats = false;
   bool LexMasmIntegers = false;
+  bool LexMasmStrings = false;
   bool UseMasmDefaultRadix = false;
   unsigned DefaultRadix = 10;
   AsmCommentConsumer *CommentConsumer = nullptr;
 
   MCAsmLexer();
 
   virtual AsmToken LexToken() = 0;
 
public:
   /// assume decimal unless prefixed (e.g., 0x2c [hex], 077 [octal]).
   void useMasmDefaultRadix(bool V) { UseMasmDefaultRadix = V; }
 
   unsigned getMasmDefaultRadix() const { return DefaultRadix; }
   void setMasmDefaultRadix(unsigned Radix) { DefaultRadix = Radix; }
 
   /// Set whether to lex masm-style hex float literals, such as 3f800000r.
   void setLexMasmHexFloats(bool V) { LexMasmHexFloats = V; }
+
+  /// Set whether to lex masm-style string literals, such as 'Can''t find file'
+  /// and "This ""value"" not found".
+  void setLexMasmStrings(bool V) { LexMasmStrings = V; }
 };
 
 } // end namespace llvm
 
 #endif // LLVM_MC_MCPARSER_MCASMLEXER_H

llvm/lib/MC/MCParser/AsmLexer.cpp

 }
 
 int AsmLexer::getNextChar() {
   if (CurPtr == CurBuf.end())
     return EOF;
   return (unsigned char)*CurPtr++;
 }
 
+int AsmLexer::peekNextChar() {
+  if (CurPtr == CurBuf.end())
+    return EOF;
+  return (unsigned char)*CurPtr;
+}
+
 /// The leading integral digit sequence and dot should have already been
 /// consumed, some or all of the fractional digit sequence *can* have been
 /// consumed.
 AsmToken AsmLexer::LexFloatLiteral() {
   // Skip the fractional digit sequence.
   while (isDigit(*CurPtr))
     ++CurPtr;
 
AsmToken AsmLexer::LexDigit() {
 
   return intToken(Result, Value);
 }
 
 /// LexSingleQuote: Integer: 'b'
 AsmToken AsmLexer::LexSingleQuote() {
   int CurChar = getNextChar();
 
+  if (LexMasmStrings) {
+    while (CurChar != EOF) {
+      if (CurChar != '\'') {
+        CurChar = getNextChar();
+      } else if (peekNextChar() == '\'') {
+        // In MASM single-quote strings, doubled single-quotes mean an escaped
+        // single quote, so should be lexed in.
+        getNextChar();
+        CurChar = getNextChar();
+      } else {
+        break;
+      }
+    }
+    if (CurChar == EOF)
+      return ReturnError(TokStart, "unterminated string constant");
+    return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
+  }
+
   if (CurChar == '\\')
     CurChar = getNextChar();
 
   if (CurChar == EOF)
     return ReturnError(TokStart, "unterminated single quote");
 
   CurChar = getNextChar();
 
if (Res.startswith("\'\\")) {
     Value = TokStart[1];
 
   return AsmToken(AsmToken::Integer, Res, Value);
 }
 
 /// LexQuote: String: "..."
 AsmToken AsmLexer::LexQuote() {
   int CurChar = getNextChar();
+  if (LexMasmStrings) {
+    while (CurChar != EOF) {
+      if (CurChar != '"') {
+        CurChar = getNextChar();
+      } else if (peekNextChar() == '"') {
+        // In MASM double-quoted strings, doubled double-quotes mean an escaped
+        // double quote, so should be lexed in.
+        getNextChar();
+        CurChar = getNextChar();
+      } else {
+        break;
+      }
+    }
+    if (CurChar == EOF)
+      return ReturnError(TokStart, "unterminated string constant");
+    return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
+  }
+
   // TODO: does gas allow multiline string constants?
   while (CurChar != '"') {
     if (CurChar == '\\') {
       // Allow \", etc.
       CurChar = getNextChar();
     }
 
     if (CurChar == EOF)
       return ReturnError(TokStart, "unterminated string constant");
 
     CurChar = getNextChar();
   }
 
   return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
 }
 
 StringRef AsmLexer::LexUntilEndOfStatement() {

llvm/lib/MC/MCParser/MasmParser.cpp

bool MasmParser::parseDirectiveEquate(StringRef IDVal, StringRef Name,
   return false;
 }
 
 bool MasmParser::parseEscapedString(std::string &Data) {
   if (check(getTok().isNot(AsmToken::String), "expected string"))
     return true;
 
   Data = "";
+  char Quote = getTok().getString().front();
   StringRef Str = getTok().getStringContents();
-  for (unsigned i = 0, e = Str.size(); i != e; ++i) {
-    if (Str[i] != '\\') {
-      Data += Str[i];
-      continue;
-    }
-
-    // Recognize escaped characters. Note that this escape semantics currently
-    // loosely follows Darwin 'as'.
-    ++i;
-    if (i == e)
-      return TokError("unexpected backslash at end of string");
-
-    // Recognize hex sequences similarly to GNU 'as'.
-    if (Str[i] == 'x' || Str[i] == 'X') {
-      size_t length = Str.size();
-      if (i + 1 >= length || !isHexDigit(Str[i + 1]))
-        return TokError("invalid hexadecimal escape sequence");
-
-      // Consume hex characters. GNU 'as' reads all hexadecimal characters and
-      // then truncates to the lower 16 bits. Seems reasonable.
-      unsigned Value = 0;
-      while (i + 1 < length && isHexDigit(Str[i + 1]))
-        Value = Value * 16 + hexDigitValue(Str[++i]);
-
-      Data += (unsigned char)(Value & 0xFF);
-      continue;
-    }
-
-    // Recognize octal sequences.
-    if ((unsigned)(Str[i] - '0') <= 7) {
-      // Consume up to three octal characters.
-      unsigned Value = Str[i] - '0';
-
-      if (i + 1 != e && ((unsigned)(Str[i + 1] - '0')) <= 7) {
-        ++i;
-        Value = Value * 8 + (Str[i] - '0');
-
-        if (i + 1 != e && ((unsigned)(Str[i + 1] - '0')) <= 7) {
-          ++i;
-          Value = Value * 8 + (Str[i] - '0');
-        }
-      }
-
-      if (Value > 255)
-        return TokError("invalid octal escape sequence (out of range)");
-
-      Data += (unsigned char)Value;
-      continue;
-    }
-
-    // Otherwise recognize individual escapes.
-    switch (Str[i]) {
-    default:
-      // Just reject invalid escape sequences for now.
-      return TokError("invalid escape sequence (unrecognized character)");
-
-    case 'b': Data += '\b'; break;
-    case 'f': Data += '\f'; break;
-    case 'n': Data += '\n'; break;
-    case 'r': Data += '\r'; break;
-    case 't': Data += '\t'; break;
-    case '"': Data += '"'; break;
-    case '\\': Data += '\\'; break;
+  Data.reserve(Str.size());
+  for (int i = 0, e = Str.size(); i != e; ++i) {
+    Data.push_back(Str[i]);
+    if (Str[i] == Quote) {
+      // MASM treats doubled delimiting quotes as an escaped delimiting quote.
+      // If we're escaping the string's trailing delimiter, we're definitely
+      // missing a quotation mark.
+      if (i + 1 == Str.size())
+        return Error(getTok().getLoc(), "missing quotation mark in string");
+      if (Str[i + 1] == Quote)
     }
   }
 
   Lex();
   return false;
 }
 
 bool MasmParser::parseAngleBracketString(std::string &Data) {
bool MasmParser::emitIntValue(const MCExpr *Value, unsigned Size) {
   }
   return false;
 }
 
 bool MasmParser::parseScalarInitializer(unsigned Size,
                                         SmallVectorImpl<const MCExpr *> &Values,
                                         unsigned StringPadLength) {
   if (getTok().is(AsmToken::String)) {
-    StringRef Value = getTok().getStringContents();
+    std::string Value;
+    if (parseEscapedString(Value))
+      return true;
     if (Size == 1) {
       // Treat each character as an initializer.
       for (const char CharVal : Value)
         Values.push_back(MCConstantExpr::create(CharVal, getContext()));
 
       // Pad the string with spaces to the specified length.
       for (size_t i = Value.size(); i < StringPadLength; ++i)
         Values.push_back(MCConstantExpr::create(' ', getContext()));
     } else {
       // Treat the string as an initial value in big-endian representation.
       if (Value.size() > Size)
         return Error(getTok().getLoc(), "out of range literal value");
 
       uint64_t IntValue = 0;
-      for (const unsigned char CharVal : Value.bytes())
+      for (const unsigned char CharVal : Value)
         IntValue = (IntValue << 8) | CharVal;
       Values.push_back(MCConstantExpr::create(IntValue, getContext()));
     }
-    Lex();
   } else {
     const MCExpr *Value;
     if (parseExpression(Value))
       return true;
     if (getTok().is(AsmToken::Identifier) &&
         getTok().getString().equals_lower("dup")) {
       Lex(); // Eat 'dup'.
       const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value);

llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp

case AsmToken::Dollar:
         if ((Done = SM.isValidEndState()))
           break;
         return Error(Tok.getLoc(), "unknown token in expression");
       }
       LLVM_FALLTHROUGH;
     case AsmToken::At:
     case AsmToken::String:
     case AsmToken::Identifier: {
+      if (Parser.isParsingMasm() && Tok.is(AsmToken::String)) {
+        // Single-character strings should be treated as integer constants. This
+        // includes MASM escapes for quotes.
+        char Quote = Tok.getString().front();
+        StringRef Contents = Tok.getStringContents();
+        if (Contents.size() == 1 || Contents == std::string(2, Quote)) {
+          if (SM.onInteger(Contents.front(), ErrMsg))
+            return Error(Tok.getLoc(), ErrMsg);
+          break;
+        }
+      }
       SMLoc IdentLoc = Tok.getLoc();
       StringRef Identifier = Tok.getString();
       UpdateLocLex = false;
       if (Parser.isParsingMasm()) {
         size_t DotOffset = Identifier.find_first_of('.');
         if (DotOffset != StringRef::npos) {
           consumeToken();
           StringRef LHS = Identifier.slice(0, DotOffset);

llvm/test/tools/llvm-ml/strings.test

+# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+
+.data
+
+dq_single_character BYTE "a"
+; CHECK-LABEL: dq_single_character:
+; CHECK-NEXT: .byte 97
+; CHECK-NOT: .byte
+
+dq_join BYTE "ab", "cd"
+; CHECK-LABEL: dq_join:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+dq_quote_escape BYTE "ab""""cd"
+; Intended result: ab""cd
+; CHECK-LABEL: dq_quote_escape:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+dq_single_quote BYTE "ab''''cd"
+; Intended result: ab''''cd
+; CHECK-LABEL: dq_single_quote:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+sq_single_character BYTE 'a'
+; CHECK-LABEL: sq_single_character:
+; CHECK-NEXT: .byte 97
+; CHECK-NOT: .byte
+
+sq_join BYTE 'ab', 'cd'
+; CHECK-LABEL: sq_join:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+sq_quote_escape BYTE 'ab''''cd'
+; Intended result: ab''cd
+; CHECK-LABEL: sq_quote_escape:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+sq_double_quote BYTE 'ab""""cd'
+; Intended result: ab""""cd
+; CHECK-LABEL: sq_double_quote:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+mixed_quotes_join BYTE "a'b", 'c"d'
+; Intended result: a'bc"d
+; CHECK-LABEL: mixed_quotes_join:
+; CHECK-NEXT: .byte 97
+; CHECK-NEXT: .byte 39
+; CHECK-NEXT: .byte 98
+; CHECK-NEXT: .byte 99
+; CHECK-NEXT: .byte 34
+; CHECK-NEXT: .byte 100
+; CHECK-NOT: .byte
+
+.code
+
+sq_char_test PROC
+; CHECK-LABEL: sq_char_test:
+
+  mov eax, 'a'
+; CHECK: mov eax, 97
+
+  ret
+sq_char_test ENDP
+
+dq_char_test PROC
+; CHECK-LABEL: dq_char_test:
+
+  mov eax, "b"
+; CHECK: mov eax, 98
+
+  ret
+dq_char_test ENDP
+
+end

llvm/test/tools/llvm-ml/struct.test

 ; BYTE "abcde", plus alignment padding
 ; CHECK-NEXT: .byte 97
 ; CHECK-NEXT: .byte 98
 ; CHECK-NEXT: .byte 99
 ; CHECK-NEXT: .byte 100
 ; CHECK-NEXT: .byte 101
 ; CHECK-NEXT: .zero 1
 
-t2 FOOBAR <"gh",,<10,11>,<12>,"ijk">
+t2 FOOBAR <"gh",,<10,11>,<12>,'ijk'>
 
 ; CHECK: t2:
 ;
 ; BYTE "gh", padded with " ", plus alignment padding
 ; CHECK-NEXT: .byte 103
 ; CHECK-NEXT: .byte 104
 ; CHECK-NEXT: .byte 32
 ; CHECK-NEXT: .zero 1

llvm/tools/llvm-ml/llvm-ml.cpp

 }
 
 static int AsLexInput(SourceMgr &SrcMgr, MCAsmInfo &MAI, raw_ostream &OS) {
   AsmLexer Lexer(MAI);
   Lexer.setBuffer(SrcMgr.getMemoryBuffer(SrcMgr.getMainFileID())->getBuffer());
   Lexer.setLexMasmIntegers(true);
   Lexer.useMasmDefaultRadix(true);
   Lexer.setLexMasmHexFloats(true);
+  Lexer.setLexMasmStrings(true);
 
   bool Error = false;
   while (Lexer.Lex().isNot(AsmToken::Eof)) {
     Lexer.getTok().dump(OS);
     OS << "\n";
     if (Lexer.getTok().getKind() == AsmToken::Error)
       Error = true;
   }
if (!TAP) {
     return 1;
   }
 
   Parser->setShowParsedOperands(ShowInstOperands);
   Parser->setTargetParser(*TAP);
   Parser->getLexer().setLexMasmIntegers(true);
   Parser->getLexer().useMasmDefaultRadix(true);
   Parser->getLexer().setLexMasmHexFloats(true);
+  Parser->getLexer().setLexMasmStrings(true);
 
   int Res = Parser->Run(/*NoInitialTextSection=*/true);
 
   return Res;
 }
 
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);