Ignore Lexing errors in macro body definitions

[niravd]

Summary

Do not issue lexing errors found during the parsing of macro body
definitions and parseIdentifier function in AsmParser. This changes the
Parser to not issue a lexing error when we reach an error, but rather
when it is consumed allowing us time to examine and recover from an
error.

As a result, of this, we stop issuing a both lexing error and a parsing
error in floating-literals test. Minor tweak to parseDirectiveRealValue
to favor more meaningful lexing error over less helpful parse error.

[niravd]

Add missing test case for macro parsing

[rnk]

lgtm I spent a fair amount of time thinking about how to do this better, but let's go with the solution we have now.

[niravd]

Merge in follow up fix minor error cleanup to make work with tip

[niravd]

Merged in subsequent error cleanup patch as accepted patch breaks in tip. Can I get another LGTM?

In D20535#444732, @rnk wrote:

lgtm I spent a fair amount of time thinking about how to do this better, but let's go with the solution we have now.

[rnk]

Still looks good. In general, you don't need explicit approval for things like fixing the build, merging patches, fixing broken tests, etc.

lib/MC/MCParser/AsmParser.cpp

if (!ParsingInlineAsm && !getStreamer().getCurrentSection().first) {
     TokError("expected section directive before assembly directive");
     Out.InitSections(false);
   }
 }
 
 /// \brief Throw away the rest of the line for testing purposes.
 void AsmParser::eatToEndOfStatement() {
   while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.isNot(AsmToken::Eof))
-    Lex();
+    Lexer.Lex();
 
   // Eat EOL.
   if (Lexer.is(AsmToken::EndOfStatement))
-    Lex();
+    Lexer.Lex();
 }
 
 StringRef AsmParser::parseStringToEndOfStatement() {
   const char *Start = getTok().getLoc().getPointer();
 
   while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.isNot(AsmToken::Eof))
     Lex();
 
bool AsmParser::parseIdentifier(StringRef &Res) {
   // allow things like '.globl $foo' and '.def @feat.00', which would normally be
   // separate tokens. At this level, we have already lexed so we cannot (currently)
   // handle this as a context dependent token, instead we detect adjacent tokens
   // and return the combined identifier.
   if (Lexer.is(AsmToken::Dollar) || Lexer.is(AsmToken::At)) {
     SMLoc PrefixLoc = getLexer().getLoc();
 
     // Consume the prefix character, and check for a following identifier.
-    Lex();
+    Lexer.Lex();
     if (Lexer.isNot(AsmToken::Identifier))
       return true;
 
     // We have a '$' or '@' followed by an identifier, make sure they are adjacent.
     if (PrefixLoc.getPointer() + 1 != getTok().getLoc().getPointer())
       return true;
 
     // Construct the joined identifier and consume the token.
     Res =
         StringRef(PrefixLoc.getPointer(), getTok().getIdentifier().size() + 1);
-    Lex();
+    Lexer.Lex();
     return false;
   }
 
   if (Lexer.isNot(AsmToken::Identifier) && Lexer.isNot(AsmToken::String))
     return true;
 
   Res = getTok().getIdentifier();
 
-  Lex(); // Consume the identifier token.
+  Lexer.Lex(); // Consume the identifier token.
 
   return false;
 }
 
 /// parseDirectiveSet:
 ///   ::= .equ identifier ',' expression
 ///   ::= .equiv identifier ',' expression
 ///   ::= .set identifier ',' expression
bool AsmParser::parseDirectiveRealValue(const fltSemantics &Semantics) {
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
     checkForValidSection();
 
     for (;;) {
       // We don't truly support arithmetic on floating point expressions, so we
       // have to manually parse unary prefixes.
       bool IsNeg = false;
       if (getLexer().is(AsmToken::Minus)) {
-        Lex();
+        Lexer.Lex();
         IsNeg = true;
       } else if (getLexer().is(AsmToken::Plus))
-        Lex();
+        Lexer.Lex();
 
-      if (getLexer().isNot(AsmToken::Integer) &&
-          getLexer().isNot(AsmToken::Real) &&
-          getLexer().isNot(AsmToken::Identifier))
+      if (Lexer.is(AsmToken::Error))
+        return TokError(Lexer.getErr());
+      if (Lexer.isNot(AsmToken::Integer) && Lexer.isNot(AsmToken::Real) &&
+          Lexer.isNot(AsmToken::Identifier))
         return TokError("unexpected token in directive");
 
       // Convert to an APFloat.
       APFloat Value(Semantics);
       StringRef IDVal = getTok().getString();
       if (getLexer().is(AsmToken::Identifier)) {
         if (!IDVal.compare_lower("infinity") || !IDVal.compare_lower("inf"))
           Value = APFloat::getInf(Semantics);
for (;;) {
       // Consume the numeric token.
       Lex();
 
       // Emit the value as an integer.
       APInt AsInt = Value.bitcastToAPInt();
       getStreamer().EmitIntValue(AsInt.getLimitedValue(),
                                  AsInt.getBitWidth() / 8);
 
-      if (getLexer().is(AsmToken::EndOfStatement))
+      if (Lexer.is(AsmToken::EndOfStatement))
         break;
 
-      if (getLexer().isNot(AsmToken::Comma))
+      if (Lexer.isNot(AsmToken::Comma))
         return TokError("unexpected token in directive");
       Lex();
     }
   }
 
   Lex();
   return false;
 }
while (getLexer().isNot(AsmToken::EndOfStatement)) {
 
     if (getLexer().is(AsmToken::Comma))
       Lex();
   }
 
   // Eat the end of statement.
   Lex();
 
+  // Consuming deferred text, so use Lexer.Lex to ignore Lexing Errors
   AsmToken EndToken, StartToken = getTok();
   unsigned MacroDepth = 0;
-
   // Lex the macro definition.
   for (;;) {
+    // Ignore Lexing errors in macros.
+    while (Lexer.is(AsmToken::Error)) {
+      Lexer.Lex();
+    }
+
     // Check whether we have reached the end of the file.
     if (getLexer().is(AsmToken::Eof))
       return Error(DirectiveLoc, "no matching '.endmacro' in definition");
 
     // Otherwise, check whether we have reach the .endmacro.
     if (getLexer().is(AsmToken::Identifier)) {
       if (getTok().getIdentifier() == ".endm" ||
           getTok().getIdentifier() == ".endmacro") {
         if (MacroDepth == 0) { // Outermost macro.
           EndToken = getTok();
-          Lex();
+          Lexer.Lex();
           if (getLexer().isNot(AsmToken::EndOfStatement))
             return TokError("unexpected token in '" + EndToken.getIdentifier() +
                             "' directive");
           break;
         } else {
           // Otherwise we just found the end of an inner macro.
           --MacroDepth;
         }

test/MC/AsmParser/floating-literals.s

 # CHECK: .long 1094713344
 .float 0x3p2
 # CHECK: .long 872284160
 .float 0x7fp-30
 # CHECK: .long 3212836864
 .float -0x1.0p0
 
 # CHECK-ERROR: invalid hexadecimal floating-point constant: expected at least one exponent digit
-# CHECK-ERROR: unexpected token in directive
 .float 0xa.apa
 
 # CHECK-ERROR: invalid hexadecimal floating-point constant: expected at least one exponent digit
-# CHECK-ERROR: unexpected token in directive
 .double -0x1.2p+
 
 # CHECK-ERROR: invalid hexadecimal floating-point constant: expected at least one exponent digit
-# CHECK-ERROR: unexpected token in directive
 .double -0x1.2p
 
 # CHECK-ERROR: invalid hexadecimal floating-point constant: expected at least one significand digit
-# CHECK-ERROR: unexpected token in directive
 .float 0xp2
 
 # CHECK-ERROR: invalid hexadecimal floating-point constant: expected at least one significand digit
-# CHECK-ERROR: unexpected token in directive
 .float 0x.p5
 
 # CHECK-ERROR: error: invalid hexadecimal floating-point constant: expected exponent part 'p'
-# CHECK-ERROR: unexpected token in directive
 .float 0x1.2