[MC/AsmLexer] Add '?' (Question) token

[barannikov88]

Summary

'?' is a valid token in our downstream target. There seem to be no way
to do target-specific lexing, so just add make AsmParser recognize it.

[barannikov88]

Update a comment

[barannikov88]

Rearrange switch cases to match the comment

[MaskRay]

I'll be out for a few days and don't have time to read many patches today.
Do you whether GNU assembler accepts ? for any target? If not, this can be a bit strange...

[barannikov88]

In D154202#4464268, @MaskRay wrote:

I'll be out for a few days and don't have time to read many patches today.

It is not urgent and I'm not pushing you. It's just I don't know who else to ask for review, not many people have been contributing to MC layer last years.

Do you whether GNU assembler accepts ? for any target? If not, this can be a bit strange...

Last time I checked, GNU assembler didn't have dedicated lexer. It has helper routines for parsing expressions, identifiers, numbers etc. but otherwise parsing is up to the target, so I guess it can accept pretty much everything.

I wish I could do it by adding some code to target-specific parser, but this is not currently possible: unrecognized characters are replaced with AsmToken::Error by the lexer and they do not even get to MCTargetAsmParser, which is the only customization point (apart from MCAsmInfo callbacks).

[barannikov88]

Update failing unit test

[MaskRay]

In MSVC, mangled C++ names start with ? and the symbols are quoted. If your target needs ?, you can use quotes as well?

In GNU assembler, there is an undocumented MRI mode that accepts ?, but we don't support the mode.

[barannikov88]

In D154202#4464906, @MaskRay wrote:

In MSVC, mangled C++ names start with ? and the symbols are quoted. If your target needs ?, you can use quotes as well?

'?' is already allowed to be a part of an identifier, and it can also be the first character of an identifier if doesAllowQuestionAtStartOfIdentifier returns true.
I need this symbol in a different context -- it is used as ternary operator in some instructions, e.g. ".float vreg0 = not mask ? vreg1 : vreg2" (assembler syntax is "high level"-ish).
The spaces around '?' are optional.

Note that adding a dedicated token kind for '?' can't break anything, because it used to be a hard error to use it in non-identifier context. This also matches how other "special" identifier characters are handled: '$', '@' and '#'.

[MaskRay]

I agree that this is a way forward and enable disabling ? in the middle of symbol names.

(Unfortunately D72680 did not add ? at symbol start tests, and I am unclear whether ml.exe does accept these symbols.)

llvm/include/llvm/MC/MCAsmMacro.h

enum TokenKind {
     // No-value.
     EndOfStatement,
     Colon,
     Space,
     Plus, Minus, Tilde,
     Slash,     // '/'
     BackSlash, // '\'
     LParen, RParen, LBrac, RBrac, LCurly, RCurly,
-    Star, Dot, Comma, Dollar, Equal, EqualEqual,
+    Question, Star, Dot, Comma, Dollar, Equal, EqualEqual,
 
     Pipe, PipePipe, Caret,
     Amp, AmpAmp, Exclaim, ExclaimEqual, Percent, Hash,
     Less, LessEqual, LessLess, LessGreater,
     Greater, GreaterEqual, GreaterGreater, At, MinusGreater,
 
     // MIPS unary expression operators such as %neg.
     PercentCall16, PercentCall_Hi, PercentCall_Lo, PercentDtprel_Hi,

llvm/lib/MC/MCParser/AsmLexer.cpp

if (CurChar == EOF && !IsAtStartOfStatement && EndStatementAtEOF) {
     IsAtStartOfStatement = true;
     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 0));
   }
   IsAtStartOfLine = false;
   bool OldIsAtStartOfStatement = IsAtStartOfStatement;
   IsAtStartOfStatement = false;
   switch (CurChar) {
   default:
-    // Handle identifier: [a-zA-Z_.?][a-zA-Z0-9_$.@#?]*
-    if (isalpha(CurChar) || CurChar == '_' || CurChar == '.' ||
-        (MAI.doesAllowQuestionAtStartOfIdentifier() && CurChar == '?'))
+    // Handle identifier: [a-zA-Z_.$@#?][a-zA-Z0-9_.$@#?]*
+    // Whether or not the lexer accepts '$', '@', '#' and '?' at the start of
+    // an identifier is target-dependent. These characters are handled in the
+    // respective switch cases.
+    if (isalpha(CurChar) || CurChar == '_' || CurChar == '.')
       return LexIdentifier();
 
     // Unknown character, emit an error.
     return ReturnError(TokStart, "invalid character in input");
   case EOF:
     if (EndStatementAtEOF) {
       IsAtStartOfLine = true;
       IsAtStartOfStatement = true;
AsmToken AsmLexer::LexToken() {
   case ',': return AsmToken(AsmToken::Comma, StringRef(TokStart, 1));
   case '$': {
     if (LexMotorolaIntegers && isHexDigit(*CurPtr))
       return LexDigit();
     if (MAI.doesAllowDollarAtStartOfIdentifier())
       return LexIdentifier();
     return AsmToken(AsmToken::Dollar, StringRef(TokStart, 1));
   }
-  case '@': {
+  case '@':
     if (MAI.doesAllowAtAtStartOfIdentifier())
       return LexIdentifier();
     return AsmToken(AsmToken::At, StringRef(TokStart, 1));
-  }
+  case '#':
+    if (MAI.doesAllowHashAtStartOfIdentifier())
+      return LexIdentifier();
+    return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
+  case '?':
+    if (MAI.doesAllowQuestionAtStartOfIdentifier())
+      return LexIdentifier();
+    return AsmToken(AsmToken::Question, StringRef(TokStart, 1));
   case '\\': return AsmToken(AsmToken::BackSlash, StringRef(TokStart, 1));
   case '=':
     if (*CurPtr == '=') {
       ++CurPtr;
       return AsmToken(AsmToken::EqualEqual, StringRef(TokStart, 2));
     }
     return AsmToken(AsmToken::Equal, StringRef(TokStart, 1));
   case '-':
if (MAI.hasMipsExpressions()) {
         CurPtr += OperatorLength - 1;
         return AsmToken(Operator, StringRef(TokStart, OperatorLength));
       }
     }
     return AsmToken(AsmToken::Percent, StringRef(TokStart, 1));
   case '/':
     IsAtStartOfStatement = OldIsAtStartOfStatement;
     return LexSlash();
-  case '#': {
-    if (MAI.doesAllowHashAtStartOfIdentifier())
-      return LexIdentifier();
-    return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
-  }
   case '\'': return LexSingleQuote();
   case '"': return LexQuote();
   case '0': case '1': case '2': case '3': case '4':
   case '5': case '6': case '7': case '8': case '9':
     return LexDigit();
   case '<':
     switch (*CurPtr) {
     case '<':

llvm/lib/MC/MCParser/MCAsmLexer.cpp

void AsmToken::dump(raw_ostream &OS) const {
   case AsmToken::LessGreater:        OS << "LessGreater"; break;
   case AsmToken::LessLess:           OS << "LessLess"; break;
   case AsmToken::Minus:              OS << "Minus"; break;
   case AsmToken::MinusGreater:       OS << "MinusGreater"; break;
   case AsmToken::Percent:            OS << "Percent"; break;
   case AsmToken::Pipe:               OS << "Pipe"; break;
   case AsmToken::PipePipe:           OS << "PipePipe"; break;
   case AsmToken::Plus:               OS << "Plus"; break;
+  case AsmToken::Question:           OS << "Question"; break;
   case AsmToken::RBrac:              OS << "RBrac"; break;
   case AsmToken::RCurly:             OS << "RCurly"; break;
   case AsmToken::RParen:             OS << "RParen"; break;
   case AsmToken::Slash:              OS << "Slash"; break;
   case AsmToken::Space:              OS << "Space"; break;
   case AsmToken::Star:               OS << "Star"; break;
   case AsmToken::Tilde:              OS << "Tilde"; break;
   case AsmToken::PercentCall16:      OS << "PercentCall16"; break;

llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp

TEST_F(SystemZAsmLexerLinux, CheckDefaultQuestionAtStartOfIdentifier) {
 
   // Setup.
   setupCallToAsmParser(AsmStr);
 
   // Lex initially to get the string.
   Parser->getLexer().Lex();
 
   SmallVector<AsmToken::TokenKind> ExpectedTokens(
-      {AsmToken::Error, AsmToken::Identifier, AsmToken::EndOfStatement,
+      {AsmToken::Question, AsmToken::Identifier, AsmToken::EndOfStatement,
        AsmToken::Eof});
   lexAndCheckTokens(AsmStr, ExpectedTokens);
 }
 
 TEST_F(SystemZAsmLexerLinux, CheckDefaultAtAtStartOfIdentifier) {
   StringRef AsmStr = "@@lh1?23";
 
   // Setup.