[AsmParser][SystemZ][z/OS] Add support to AsmLexer to accept HLASM style integers

[anirudhp]

Summary

• Add support for HLASM style integers. These are the decimal integers [0-9].
• HLASM does not support the additional prefixed integers like, 0b, 0x, octal integers and Masm style integers.
• To achieve this, a field LexHLASMStyleIntegers (similar to the LexMasmStyleIntegers field) is introduced in MCAsmLexer.h as well as a corresponding setter.

Note: This field could also go into MCAsmInfo.h. I used the previous precedent set by the LexMasmIntegers field.

Depends on https://reviews.llvm.org/D99286

[abhina.sreeskantharajan]

I got some very minor nits about formatting, but overall it looks good

llvm/lib/MC/MCParser/AsmLexer.cpp

325

nit: I think we should stick to either HLASM-style or HLASM-flavour for consistency and not both.

331

nit: comment style should be like the following to be clang-tidy friendly https://clang.llvm.org/extra/clang-tidy/checks/bugprone-argument-comment.html

llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp

389

extra space here

[anirudhp]

• Addressed comments (Minor formatting changes)

[anirudhp]

• Rebase on latest master

[epastor]

llvm/lib/MC/MCParser/AsmLexer.cpp

329–330

Can we save a lot of code here by removing this whole segment and making the decimal parsing below work even with leading 0 when LexHLASMIntegers is set?

Right now, this seems very redundant.

[anirudhp]

• Addressed Comments
• Moved logic for LexHLASMIntegers to the main block where decimal integers are parsed.

[anirudhp]

llvm/lib/MC/MCParser/AsmLexer.cpp

329–330

Done! Thanks for the hint!

[epastor]

llvm/lib/MC/MCParser/AsmLexer.cpp

462

Should this be (LexHLASMIntegers || CurPtr[-1] != 0) || CurPtr[0] == '.'? The parentheses may be relevant here, since you probably still want floating point literals starting or ending with '.' to be handled.

[epastor]

llvm/lib/MC/MCParser/AsmLexer.cpp

462

I withdraw this comment... I think I thought the latter || was an &&. Apologies!

[anirudhp]

• Appeasing clang-tidy (rename of variable from camelCase to CamelCase)

[anirudhp]

• Rebase on latest master

[anirudhp]

• Rebase on latest master x2 (since the macho tests seem to be fixed)

llvm/include/llvm/MC/MCParser/MCAsmLexer.h

protected: // Can only create subclasses.
   bool AllowHashInIdentifier = false;
   bool IsAtStartOfStatement = true;
   bool LexMasmHexFloats = false;
   bool LexMasmIntegers = false;
   bool LexMasmStrings = false;
   bool LexMotorolaIntegers = false;
   bool UseMasmDefaultRadix = false;
   unsigned DefaultRadix = 10;
+  bool LexHLASMIntegers = false;
   AsmCommentConsumer *CommentConsumer = nullptr;
 
   MCAsmLexer();
 
   virtual AsmToken LexToken() = 0;
 
   void SetError(SMLoc errLoc, const std::string &err) {
     ErrLoc = errLoc;
public:
 
   /// Set whether to lex masm-style string literals, such as 'Can''t find file'
   /// and "This ""value"" not found".
   void setLexMasmStrings(bool V) { LexMasmStrings = V; }
 
   /// Set whether to lex Motorola-style integer literals, such as $deadbeef or
   /// %01010110.
   void setLexMotorolaIntegers(bool V) { LexMotorolaIntegers = V; }
+
+  /// Set whether to lex HLASM-flavour integers. For now this is only [0-9]*
+  void setLexHLASMIntegers(bool V) { LexHLASMIntegers = V; }
 };
 
 } // end namespace llvm
 
 #endif // LLVM_MC_MCPARSER_MCASMLEXER_H

llvm/lib/MC/MCParser/AsmLexer.cpp

 /// LexDigit: First character is [0-9].
 ///   Local Label: [0-9][:]
 ///   Forward/Backward Label: [0-9][fb]
 ///   Binary integer: 0b[01]+
 ///   Octal integer: 0[0-7]+
 ///   Hex integer: 0x[0-9a-fA-F]+ or [0x]?[0-9][0-9a-fA-F]*[hH]
 ///   Decimal integer: [1-9][0-9]*
 AsmToken AsmLexer::LexDigit() {
   // MASM-flavor binary integer: [01]+[yY] (if DefaultRadix < 16, [bByY])
   // MASM-flavor octal integer: [0-7]+[oOqQ]
   // MASM-flavor decimal integer: [0-9]+[tT] (if DefaultRadix < 16, [dDtT])
   // MASM-flavor hexadecimal integer: [0-9][0-9a-fA-F]*[hH]
   if (LexMasmIntegers && isdigit(CurPtr[-1])) {
     const char *FirstNonBinary =
         (CurPtr[-1] != '0' && CurPtr[-1] != '1') ? CurPtr - 1 : nullptr;
     const char *FirstNonDecimal =
         (CurPtr[-1] < '0' || CurPtr[-1] > '9') ? CurPtr - 1 : nullptr;
     const char *OldCurPtr = CurPtr;
     while (isHexDigit(*CurPtr)) {
       switch (*CurPtr) {
       default:
         if (!FirstNonDecimal) {
           FirstNonDecimal = CurPtr;
if (LexMotorolaIntegers && CurPtr[-1] == '%') {
     APInt Result(128, 0);
     if (StringRef(NumStart, CurPtr - NumStart).getAsInteger(2, Result))
       return ReturnError(TokStart, "invalid binary number");
 
     return intToken(StringRef(TokStart, CurPtr - TokStart), Result);
   }
 
   // Decimal integer: [1-9][0-9]*
-  if (CurPtr[-1] != '0' || CurPtr[0] == '.') {
+  // HLASM-flavour decimal integer: [0-9][0-9]*
+  // FIXME: Later on, support for fb for HLASM has to be added in
+  // as they probably would be needed for asm goto
+  if (LexHLASMIntegers || CurPtr[-1] != '0' || CurPtr[0] == '.') {
     unsigned Radix = doHexLookAhead(CurPtr, 10, LexMasmIntegers);
-    bool isHex = Radix == 16;
+
+    if (!LexHLASMIntegers) {
+      bool IsHex = Radix == 16;
       // Check for floating point literals.
-    if (!isHex && (*CurPtr == '.' || *CurPtr == 'e' || *CurPtr == 'E')) {
+      if (!IsHex && (*CurPtr == '.' || *CurPtr == 'e' || *CurPtr == 'E')) {
         if (*CurPtr == '.')
           ++CurPtr;
         return LexFloatLiteral();
       }
+    }
 
     StringRef Result(TokStart, CurPtr - TokStart);
 
     APInt Value(128, 0, true);
-    if (Result.getAsInteger(Radix, Value)) {
+    if (Result.getAsInteger(Radix, Value))
       return ReturnError(TokStart, "invalid " + radixName(Radix) + " number");
-    }
 
+    if (!LexHLASMIntegers)
       // The darwin/x86 (and x86-64) assembler accepts and ignores type
       // suffices on integer literals.
       SkipIgnoredIntegerSuffix(CurPtr);
 
     return intToken(Result, Value);
   }
 
   if (!LexMasmIntegers && ((*CurPtr == 'b') || (*CurPtr == 'B'))) {
     ++CurPtr;
     // See if we actually have "0b" as part of something like "jmp 0b\n"
     if (!isDigit(CurPtr[0])) {

llvm/unittests/MC/SystemZ/SystemZAsmLexerTest.cpp

void lexAndCheckTokens(StringRef AsmStr,
     // Get reference to AsmLexer.
     MCAsmLexer &Lexer = Parser->getLexer();
     // Loop through all expected tokens checking one by one.
     for (size_t I = 0; I < ExpectedTokens.size(); ++I) {
       EXPECT_EQ(Lexer.getTok().getKind(), ExpectedTokens[I]);
       Lexer.Lex();
     }
   }
+
+  void lexAndCheckIntegerTokensAndValues(StringRef AsmStr,
+                                         SmallVector<int64_t> ExpectedValues) {
+    // Get reference to AsmLexer.
+    MCAsmLexer &Lexer = Parser->getLexer();
+    // Loop through all expected tokens and expected values.
+    for (size_t I = 0; I < ExpectedValues.size(); ++I) {
+      // Skip any EndOfStatement tokens, we're not concerned with them.
+      if (Lexer.getTok().getKind() == AsmToken::EndOfStatement)
+        continue;
+      EXPECT_EQ(Lexer.getTok().getKind(), AsmToken::Integer);
+      EXPECT_EQ(Lexer.getTok().getIntVal(), ExpectedValues[I]);
+      Lexer.Lex();
+    }
+  }
 };
 
 TEST_F(SystemZAsmLexerTest, CheckDontRestrictCommentStringToStartOfStatement) {
   StringRef AsmStr = "jne #-4";
 
   // Setup.
   setupCallToAsmParser(AsmStr);
 
TEST_F(SystemZAsmLexerTest, CheckStrictCommentString5) {
   ExpectedTokens.push_back(AsmToken::Identifier);     // "abc"
   ExpectedTokens.push_back(AsmToken::EndOfStatement); // "\n"
   ExpectedTokens.push_back(AsmToken::Slash);          // "/"
   ExpectedTokens.push_back(AsmToken::EndOfStatement); // "* def */// xyz"
   ExpectedTokens.push_back(AsmToken::Eof);
 
   lexAndCheckTokens(AsmStr, ExpectedTokens);
 }
+
+TEST_F(SystemZAsmLexerTest, CheckValidHLASMIntegers) {
+  StringRef AsmStr = "123\n000123\n1999\n007\n12300\n12021\n";
+  // StringRef AsmStr = "123";
+  // Setup.
+  setupCallToAsmParser(AsmStr);
+  Parser->getLexer().setLexHLASMIntegers(true);
+
+  // Lex initially to get the string.
+  Parser->getLexer().Lex();
+
+  // SmallVector<int64_t> ExpectedValues({123});
+  SmallVector<int64_t> ExpectedValues({123, 123, 1999, 7, 12300, 12021});
+  lexAndCheckIntegerTokensAndValues(AsmStr, ExpectedValues);
+}
+
+TEST_F(SystemZAsmLexerTest, CheckInvalidHLASMIntegers) {
+  StringRef AsmStr = "0b0101\n0xDEADBEEF\nfffh\n.133\n";
+
+  // Setup.
+  setupCallToAsmParser(AsmStr);
+  Parser->getLexer().setLexHLASMIntegers(true);
+
+  // Lex initially to get the string.
+  Parser->getLexer().Lex();
+
+  SmallVector<AsmToken::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(AsmToken::Integer);        // "0"
+  ExpectedTokens.push_back(AsmToken::Identifier);     // "b0101"
+  ExpectedTokens.push_back(AsmToken::EndOfStatement); // "\n"
+  ExpectedTokens.push_back(AsmToken::Integer);        // "0"
+  ExpectedTokens.push_back(AsmToken::Identifier);     // "xDEADBEEF"
+  ExpectedTokens.push_back(AsmToken::EndOfStatement); // "\n"
+  ExpectedTokens.push_back(AsmToken::Identifier);     // "fffh"
+  ExpectedTokens.push_back(AsmToken::EndOfStatement); // "\n"
+  ExpectedTokens.push_back(AsmToken::Real);           // ".133"
+  ExpectedTokens.push_back(AsmToken::EndOfStatement); // "\n"
+  ExpectedTokens.push_back(AsmToken::Eof);
+  lexAndCheckTokens(AsmStr, ExpectedTokens);
+}
+
+TEST_F(SystemZAsmLexerTest, CheckDefaultIntegers) {
+  StringRef AsmStr = "0b0101\n0xDEADBEEF\nfffh\n";
+
+  // Setup.
+  setupCallToAsmParser(AsmStr);
+
+  // Lex initially to get the string.
+  Parser->getLexer().Lex();
+
+  SmallVector<int64_t> ExpectedValues({5, 0xDEADBEEF, 0xFFF});
+  lexAndCheckIntegerTokensAndValues(AsmStr, ExpectedValues);
+}
+
+TEST_F(SystemZAsmLexerTest, CheckDefaultFloats) {
+  StringRef AsmStr = "0.333\n1.3\n2.5\n3.0\n";
+
+  // Setup.
+  setupCallToAsmParser(AsmStr);
+
+  // Lex initially to get the string.
+  Parser->getLexer().Lex();
+
+  SmallVector<AsmToken::TokenKind> ExpectedTokens;
+
+  for (int I = 0; I < 4; ++I)
+    ExpectedTokens.insert(ExpectedTokens.begin(),
+                          {AsmToken::Real, AsmToken::EndOfStatement});
+
+  ExpectedTokens.push_back(AsmToken::Eof);
+  lexAndCheckTokens(AsmStr, ExpectedTokens);
+}
 } // end anonymous namespace