Refactor and cleanup Assembly Parsing / Lexing

[niravd]

Summary

Add explicit Comment Token in Assembly Lexing for future support for
outputting explicit comments from inline assembly. As part of this,
CPPHash Directives are now explicitly distinguished from Hash line
comments in Lexer.

Line comments are recorded as EndOfStatement tokens, not Comment tokens
to simplify compatibility with current TargetParsers. This slightly
complicates comment output.

This remove all lexing tasks out of the parser, does minor cleanup
to remove extraneous newlines Asm Output, and some improvements white
space handling.

[niravd]

Modifications to keep line comments in correct place with TargetParser

[niravd]

Update to fix CPP Hash Directive parsing and add testcase

[rnk]

Overall seems like a good cleanup of a janky parser. =/

include/llvm/MC/MCParser/AsmLexer.h

32–33

These should both follow LLVM style, i.e. leading upper case for variables & fields.

lib/MC/MCParser/AsmLexer.cpp

53

Removing this attempt to optimize seems fine.

514

Follow the LLVM naming conventions please.

lib/MC/MCParser/AsmParser.cpp

627

Really it skips them, it doesn't parse them.

[niravd]

Revert enum as requested in D20020, fix LLVM style issues

[niravd]

Everything appears to be resolved. Can I get a LGTM?

[rnk]

lgtm

[loladiro]

Please note https://llvm.org/bugs/show_bug.cgi?id=28921, which has a testcase bisected to this revision.

include/llvm/MC/MCParser/AsmLexer.h

 class MCAsmInfo;
 
 /// AsmLexer - Lexer class for assembly files.
 class AsmLexer : public MCAsmLexer {
   const MCAsmInfo &MAI;
 
   const char *CurPtr;
   StringRef CurBuf;
   bool isAtStartOfLine;
 
   void operator=(const AsmLexer&) = delete;
   AsmLexer(const AsmLexer&) = delete;
 
 protected:
   /// LexToken - Read the next token and return its code.
   AsmToken LexToken() override;
 
 public:
   AsmLexer(const MCAsmInfo &MAI);
   ~AsmLexer() override;
 
   void setBuffer(StringRef Buf, const char *ptr = nullptr);
 
   StringRef LexUntilEndOfStatement() override;
-  StringRef LexUntilEndOfLine();
 
   size_t peekTokens(MutableArrayRef<AsmToken> Buf,
                     bool ShouldSkipSpace = true) override;
 
-  bool isAtStartOfComment(const char *Ptr);
-  bool isAtStatementSeparator(const char *Ptr);
-
   const MCAsmInfo &getMAI() const { return MAI; }
 
 private:
+  bool isAtStartOfComment(const char *Ptr);
+  bool isAtStatementSeparator(const char *Ptr);
   int getNextChar();
   AsmToken ReturnError(const char *Loc, const std::string &Msg);
 
   AsmToken LexIdentifier();
   AsmToken LexSlash();
   AsmToken LexLineComment();
   AsmToken LexDigit();
   AsmToken LexSingleQuote();
   AsmToken LexQuote();
   AsmToken LexFloatLiteral();
   AsmToken LexHexFloatLiteral(bool NoIntDigits);
+
+  StringRef LexUntilEndOfLine();
 };
 
 } // end namespace llvm
 
 #endif

include/llvm/MC/MCParser/MCAsmLexer.h

 
 namespace llvm {
 
 /// Target independent representation for an assembler token.
 class AsmToken {
 public:
   enum TokenKind {
     // Markers
-    Eof, Error,
+    Eof,
+    Error,
 
     // String values.
     Identifier,
     String,
 
     // Integer values.
     Integer,
     BigNum, // larger than 64 bits
 
     // Real values.
     Real,
 
+    // Comments
+    Comment,
+    HashDirective,
     // No-value.
     EndOfStatement,
     Colon,
     Space,
-    Plus, Minus, Tilde,
-    Slash,    // '/'
+    Plus,
+    Minus,
+    Tilde,
+    Slash,     // '/'
     BackSlash, // '\'
-    LParen, RParen, LBrac, RBrac, LCurly, RCurly,
-    Star, Dot, Comma, Dollar, Equal, EqualEqual,
-
-    Pipe, PipePipe, Caret,
-    Amp, AmpAmp, Exclaim, ExclaimEqual, Percent, Hash,
-    Less, LessEqual, LessLess, LessGreater,
-    Greater, GreaterEqual, GreaterGreater, At
+    LParen,
+    RParen,
+    LBrac,
+    RBrac,
+    LCurly,
+    RCurly,
+    Star,
+    Dot,
+    Comma,
+    Dollar,
+    Equal,
+    EqualEqual,
+
+    Pipe,
+    PipePipe,
+    Caret,
+    Amp,
+    AmpAmp,
+    Exclaim,
+    ExclaimEqual,
+    Percent,
+    Hash,
+    Less,
+    LessEqual,
+    LessLess,
+    LessGreater,
+    Greater,
+    GreaterEqual,
+    GreaterGreater,
+    At
   };
 
 private:
   TokenKind Kind;
 
   /// A reference to the entire token contents; this is always a pointer into
   /// a memory buffer owned by the source manager.
   StringRef Str;
public:
 
   /// Consume the next token from the input stream and return it.
   ///
   /// The lexer will continuosly return the end-of-file token once the end of
   /// the main input file has been reached.
   const AsmToken &Lex() {
     assert(!CurTok.empty());
     CurTok.erase(CurTok.begin());
+    // Always place in front as LexToken may generate multiple tokens via UnLex.
     if (CurTok.empty())
-      CurTok.emplace_back(LexToken());
+      CurTok.insert(CurTok.begin(), LexToken());
     return CurTok.front();
   }
 
   void UnLex(AsmToken const &Token) {
     CurTok.insert(CurTok.begin(), Token);
   }
 
   virtual StringRef LexUntilEndOfStatement() = 0;

lib/MC/MCParser/AsmLexer.cpp

 /// location.  This is defined to always return AsmToken::Error.
 AsmToken AsmLexer::ReturnError(const char *Loc, const std::string &Msg) {
   SetError(SMLoc::getFromPointer(Loc), Msg);
 
   return AsmToken(AsmToken::Error, StringRef(Loc, 0));
 }
 
 int AsmLexer::getNextChar() {
-  char CurChar = *CurPtr++;
-  switch (CurChar) {
-  default:
-    return (unsigned char)CurChar;
-  case 0:
-    // A nul character in the stream is either the end of the current buffer or
-    // a random nul in the file.  Disambiguate that here.
-    if (CurPtr - 1 != CurBuf.end())
-      return 0;  // Just whitespace.
-
-    // Otherwise, return end of file.
-    --CurPtr;  // Another call to lex will return EOF again.
+  if (CurPtr == CurBuf.end())
     return EOF;
-  }
+  return (unsigned char)*CurPtr++;
 }
 
 /// LexFloatLiteral: [0-9]*[.][0-9]*([eE][+-]?[0-9]*)?
 ///
 /// The leading integral digit sequence and dot should have already been
 /// consumed, some or all of the fractional digit sequence *can* have been
 /// consumed.
 AsmToken AsmLexer::LexFloatLiteral() {

 AsmToken AsmLexer::LexSlash() {
   switch (*CurPtr) {
   case '*':
     break; // C style comment.
   case '/':
     ++CurPtr;
     return LexLineComment();
   default:
+    isAtStartOfLine = false;
     return AsmToken(AsmToken::Slash, StringRef(CurPtr - 1, 1));
   }
 
   // C Style comment.
   ++CurPtr;  // skip the star.
-  while (1) {
-    int CurChar = getNextChar();
-    switch (CurChar) {
-    case EOF:
-      return ReturnError(TokStart, "unterminated comment");
+  while (CurPtr != CurBuf.end()) {
+    switch (*CurPtr++) {
     case '*':
       // End of the comment?
-      if (CurPtr[0] != '/') break;
-
+      if (*CurPtr != '/')
+        break;
       ++CurPtr;   // End the */.
-      return LexToken();
+      return AsmToken(AsmToken::Comment,
+                      StringRef(TokStart, CurPtr - TokStart));
     }
   }
+  return ReturnError(TokStart, "unterminated comment");
 }
 
 /// LexLineComment: Comment: #[^\n]*
 ///                        : //[^\n]*
 AsmToken AsmLexer::LexLineComment() {
-  // FIXME: This is broken if we happen to a comment at the end of a file, which
-  // was .included, and which doesn't end with a newline.
+  // Mark This as an end of statement with a body of the
+  // comment. While it would be nicer to leave this two tokens,
+  // backwards compatability with TargetParsers makes keeping this in this form
+  // better.
   int CurChar = getNextChar();
   while (CurChar != '\n' && CurChar != '\r' && CurChar != EOF)
     CurChar = getNextChar();
 
-  if (CurChar == EOF)
-    return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
-  return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 0));
+  // Whis is a whole line comment. leave newline
+  if (isAtStartOfLine)
+    return AsmToken(AsmToken::EndOfStatement,
+                    StringRef(TokStart, CurPtr - TokStart));
+  isAtStartOfLine = true;
+
+  return AsmToken(AsmToken::EndOfStatement,
+                  StringRef(TokStart, CurPtr - 1 - TokStart));
 }
 
 static void SkipIgnoredIntegerSuffix(const char *&CurPtr) {
   // Skip ULL, UL, U, L and LL suffices.
   if (CurPtr[0] == 'U')
     ++CurPtr;
   if (CurPtr[0] == 'L')
     ++CurPtr;
AsmToken AsmLexer::LexQuote() {
   return AsmToken(AsmToken::String, StringRef(TokStart, CurPtr - TokStart));
 }
 
 StringRef AsmLexer::LexUntilEndOfStatement() {
   TokStart = CurPtr;
 
   while (!isAtStartOfComment(CurPtr) &&     // Start of line comment.
          !isAtStatementSeparator(CurPtr) && // End of statement marker.
-         *CurPtr != '\n' && *CurPtr != '\r' &&
-         (*CurPtr != 0 || CurPtr != CurBuf.end())) {
+         *CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
     ++CurPtr;
   }
   return StringRef(TokStart, CurPtr-TokStart);
 }
 
 StringRef AsmLexer::LexUntilEndOfLine() {
   TokStart = CurPtr;
 
-  while (*CurPtr != '\n' && *CurPtr != '\r' &&
-         (*CurPtr != 0 || CurPtr != CurBuf.end())) {
+  while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end()) {
     ++CurPtr;
   }
   return StringRef(TokStart, CurPtr-TokStart);
 }
 
 size_t AsmLexer::peekTokens(MutableArrayRef<AsmToken> Buf,
                             bool ShouldSkipSpace) {
   const char *SavedTokStart = TokStart;
return strncmp(Ptr, MAI.getSeparatorString(),
                  strlen(MAI.getSeparatorString())) == 0;
 }
 
 AsmToken AsmLexer::LexToken() {
   TokStart = CurPtr;
   // This always consumes at least one character.
   int CurChar = getNextChar();
 
+  if (CurChar == '#' && isAtStartOfLine) {
+    // If this comment starts with a '#', this may be a cpp
+    // hash directive and otherwise a line comment.
+    AsmToken TokenBuf[2];
+    MutableArrayRef<AsmToken> Buf(TokenBuf, 2);
+    size_t num = peekTokens(Buf, true);
+    if (num == 2 && TokenBuf[0].is(AsmToken::Integer) &&
+        TokenBuf[1].is(AsmToken::String)) {
+      CurPtr = TokStart; // reset curPtr;
+      StringRef s = LexUntilEndOfLine();
+      UnLex(TokenBuf[1]);
+      UnLex(TokenBuf[0]);
+      return AsmToken(AsmToken::HashDirective, s);
+    }
+    return LexLineComment();
+  }
+
   if (isAtStartOfComment(TokStart)) {
-    // If this comment starts with a '#', then return the Hash token and let
-    // the assembler parser see if it can be parsed as a cpp line filename
-    // comment. We do this only if we are at the start of a line.
-    if (CurChar == '#' && isAtStartOfLine)
-      return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
-    isAtStartOfLine = true;
     return LexLineComment();
   }
   if (isAtStatementSeparator(TokStart)) {
     CurPtr += strlen(MAI.getSeparatorString()) - 1;
     return AsmToken(AsmToken::EndOfStatement,
                     StringRef(TokStart, strlen(MAI.getSeparatorString())));
   }
 
   // If we're missing a newline at EOF, make sure we still get an
   // EndOfStatement token before the Eof token.
   if (CurChar == EOF && !isAtStartOfLine) {
     isAtStartOfLine = true;
     return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
   }
-
+  bool OldIsAtStartOfLine = isAtStartOfLine;
   isAtStartOfLine = false;
   switch (CurChar) {
   default:
     // Handle identifier: [a-zA-Z_.][a-zA-Z0-9_$.@]*
     if (isalpha(CurChar) || CurChar == '_' || CurChar == '.')
       return LexIdentifier();
 
     // Unknown character, emit an error.
     return ReturnError(TokStart, "invalid character in input");
-  case EOF: return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
+  case EOF:
+    isAtStartOfLine = true;
+    return AsmToken(AsmToken::Eof, StringRef(TokStart, 0));
   case 0:
   case ' ':
   case '\t':
+    isAtStartOfLine = OldIsAtStartOfLine;
     if (SkipSpace) {
       // Ignore whitespace.
       return LexToken();
     } else {
       int len = 1;
       while (*CurPtr==' ' || *CurPtr=='\t') {
         CurPtr++;
         len++;
case '&':
     return AsmToken(AsmToken::Amp, StringRef(TokStart, 1));
   case '!':
     if (*CurPtr == '=') {
       ++CurPtr;
       return AsmToken(AsmToken::ExclaimEqual, StringRef(TokStart, 2));
     }
     return AsmToken(AsmToken::Exclaim, StringRef(TokStart, 1));
   case '%': return AsmToken(AsmToken::Percent, StringRef(TokStart, 1));
-  case '/': return LexSlash();
+  case '/': // preserve line starts in case of comment.
+    isAtStartOfLine = OldIsAtStartOfLine;
+    return LexSlash();
   case '#': return AsmToken(AsmToken::Hash, StringRef(TokStart, 1));
   case '\'': return LexSingleQuote();
   case '"': return LexQuote();
   case '0': case '1': case '2': case '3': case '4':
   case '5': case '6': case '7': case '8': case '9':
     return LexDigit();
   case '<':
     switch (*CurPtr) {

lib/MC/MCParser/AsmParser.cpp

public:
   void checkForValidSection() override;
   /// }
 
 private:
 
   bool parseStatement(ParseStatementInfo &Info,
                       MCAsmParserSemaCallback *SI);
   bool parseCurlyBlockScope(SmallVectorImpl<AsmRewrite>& AsmStrRewrites);
-  void eatToEndOfLine();
   bool parseCppHashLineFilenameComment(SMLoc L);
 
   void checkForBadMacro(SMLoc DirectiveLoc, StringRef Name, StringRef Body,
                         ArrayRef<MCAsmMacroParameter> Parameters);
   bool expandMacro(raw_svector_ostream &OS, StringRef Body,
                    ArrayRef<MCAsmMacroParameter> Parameters,
                    ArrayRef<MCAsmMacroArgument> A, bool EnableAtPseudoVariable,
                    SMLoc L);

 void AsmParser::jumpToLoc(SMLoc Loc, unsigned InBuffer) {
   CurBuffer = InBuffer ? InBuffer : SrcMgr.FindBufferContainingLoc(Loc);
   Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer(),
                   Loc.getPointer());
 }
 
 const AsmToken &AsmParser::Lex() {
   const AsmToken *tok = &Lexer.Lex();
+  // Parse comments here.
+  while (tok->is(AsmToken::Comment)) {
+    tok = &Lexer.Lex();
+  }
 
   if (tok->is(AsmToken::Eof)) {
     // If this is the end of an included file, pop the parent file off the
     // include stack.
     SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
     if (ParentIncludeLoc != SMLoc()) {
       jumpToLoc(ParentIncludeLoc);
-      tok = &Lexer.Lex();
+      return Lex();
     }
   }
 
   if (tok->is(AsmToken::Error))
     Error(Lexer.getErrLoc(), Lexer.getErr());
 
   return *tok;
 }
if (MAI.hasSubsectionsViaSymbols()) {
         // Variable symbols may not be marked as defined, so check those
         // explicitly. If we know it's a variable, we have a definition for
         // the purposes of this check.
         if (Sym->isTemporary() && !Sym->isVariable() && !Sym->isDefined())
           // FIXME: We would really like to refer back to where the symbol was
           // first referenced for a source location. We need to add something
           // to track that. Currently, we just point to the end of the file.
           HadError |=
-              Error(getLexer().getLoc(), "assembler local symbol '" +
-                                             Sym->getName() + "' not defined");
+              Error(getTok().getLoc(), "assembler local symbol '" +
       }
     }
 
     // Temporary symbols like the ones for directional jumps don't go in the
     // symbol table. They also need to be diagnosed in all (final) cases.
     for (std::tuple<SMLoc, CppHashInfoTy, MCSymbol *> &LocSym : DirLabels) {
       if (std::get<2>(LocSym)->isUndefined()) {
         // Reset the state of any "# line file" directives we've seen to the
if (!ParsingInlineAsm && !getStreamer().getCurrentSection().first) {
     TokError("expected section directive before assembly directive");
     Out.InitSections(false);
   }
 }
 
 /// \brief Throw away the rest of the line for testing purposes.
 void AsmParser::eatToEndOfStatement() {
   while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.isNot(AsmToken::Eof))
-    Lex();
+    Lexer.Lex();
 
   // Eat EOL.
   if (Lexer.is(AsmToken::EndOfStatement))
-    Lex();
+    Lexer.Lex();
 }
 
 StringRef AsmParser::parseStringToEndOfStatement() {
   const char *Start = getTok().getLoc().getPointer();
 
   while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.isNot(AsmToken::Eof))
-    Lex();
+    Lexer.Lex();
 
   const char *End = getTok().getLoc().getPointer();
   return StringRef(Start, End - Start);
 }
 
 StringRef AsmParser::parseStringToComma() {
   const char *Start = getTok().getLoc().getPointer();
 
   while (Lexer.isNot(AsmToken::EndOfStatement) &&
          Lexer.isNot(AsmToken::Comma) && Lexer.isNot(AsmToken::Eof))
-    Lex();
+    Lexer.Lex();
 
   const char *End = getTok().getLoc().getPointer();
   return StringRef(Start, End - Start);
 }
 
 /// \brief Parse a paren expression and return it.
 /// NOTE: This assumes the leading '(' has already been consumed.
 ///
if (parseIdentifier(Identifier)) {
         return Error(FirstTokenLoc, "invalid token in expression");
       }
     }
     // Parse symbol variant
     std::pair<StringRef, StringRef> Split;
     if (!MAI.useParensForSymbolVariant()) {
       if (FirstTokenKind == AsmToken::String) {
         if (Lexer.is(AsmToken::At)) {
-          Lexer.Lex(); // eat @
+          Lex(); // eat @
           SMLoc AtLoc = getLexer().getLoc();
           StringRef VName;
           if (parseIdentifier(VName))
             return Error(AtLoc, "expected symbol variant after '@'");
 
           Split = std::make_pair(Identifier, VName);
         }
       } else {
         Split = Identifier.split('@');
       }
     } else if (Lexer.is(AsmToken::LParen)) {
-      Lexer.Lex(); // eat (
+      Lex(); // eat '('.
       StringRef VName;
       parseIdentifier(VName);
       if (Lexer.isNot(AsmToken::RParen)) {
           return Error(Lexer.getTok().getLoc(),
                        "unexpected token in variant, expected ')'");
       }
-      Lexer.Lex(); // eat )
+      Lex(); // eat ')'.
       Split = std::make_pair(Identifier, VName);
     }
 
     EndLoc = SMLoc::getFromPointer(Identifier.end());
 
     // This is a symbol reference.
     StringRef SymbolName = Identifier;
     MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;

 }
 
 /// ParseStatement:
 ///   ::= EndOfStatement
 ///   ::= Label* Directive ...Operands... EndOfStatement
 ///   ::= Label* Identifier OperandList* EndOfStatement
 bool AsmParser::parseStatement(ParseStatementInfo &Info,
                                MCAsmParserSemaCallback *SI) {
+  // Eat initial spaces and comments
+  while (Lexer.is(AsmToken::Space))
+    Lex();
   if (Lexer.is(AsmToken::EndOfStatement)) {
+    // if this is a line comment we can drop it safely
+    if (getTok().getString().front() == '\r' ||
+        getTok().getString().front() == '\n')
       Out.AddBlankLine();
     Lex();
     return false;
   }
-
-  // Statements always start with an identifier or are a full line comment.
+  // Statements always start with an identifier.
   AsmToken ID = getTok();
   SMLoc IDLoc = ID.getLoc();
   StringRef IDVal;
   int64_t LocalLabelVal = -1;
-  // A full line comment is a '#' as the first token.
-  if (Lexer.is(AsmToken::Hash))
+  if (Lexer.is(AsmToken::HashDirective))
     return parseCppHashLineFilenameComment(IDLoc);
 
   // Allow an integer followed by a ':' as a directional local label.
   if (Lexer.is(AsmToken::Integer)) {
     LocalLabelVal = getTok().getIntVal();
     if (LocalLabelVal < 0) {
       if (!TheCondState.Ignore)
         return TokError("unexpected token at start of statement");
if (IDVal[0] == '.' && IDVal != ".") {
     case DK_ABORT:
       return parseDirectiveAbort();
     case DK_INCLUDE:
       return parseDirectiveInclude();
     case DK_INCBIN:
       return parseDirectiveIncbin();
     case DK_CODE16:
     case DK_CODE16GCC:
-      return TokError(Twine(IDVal) + " not supported yet");
+      return TokError(Twine(IDVal) +
+                      " not currently supported for this target");
     case DK_REPT:
       return parseDirectiveRept(IDLoc, IDVal);
     case DK_IRP:
       return parseDirectiveIrp(IDLoc);
     case DK_IRPC:
       return parseDirectiveIrpc(IDLoc);
     case DK_ENDR:
       return parseDirectiveEndr(IDLoc);
if (Lexer.is(AsmToken::EndOfStatement))
     Lex(); // Eat EndOfStatement following the brace
 
   // Erase the block start/end brace from the output asm string
   AsmStrRewrites.emplace_back(AOK_Skip, StartLoc, Lexer.getLoc().getPointer() -
                                                   StartLoc.getPointer());
   return true;
 }
 
-/// eatToEndOfLine uses the Lexer to eat the characters to the end of the line
-/// since they may not be able to be tokenized to get to the end of line token.
-void AsmParser::eatToEndOfLine() {
-  if (!Lexer.is(AsmToken::EndOfStatement))
-    Lexer.LexUntilEndOfLine();
-  // Eat EOL.
-  Lex();
-}
-
 /// parseCppHashLineFilenameComment as this:
 ///   ::= # number "filename"
-/// or just as a full line comment if it doesn't have a number and a string.
 bool AsmParser::parseCppHashLineFilenameComment(SMLoc L) {
   Lex(); // Eat the hash token.
-
-  if (getLexer().isNot(AsmToken::Integer)) {
-    // Consume the line since in cases it is not a well-formed line directive,
-    // as if were simply a full line comment.
-    eatToEndOfLine();
-    return false;
-  }
-
+  // Lexer only ever emits HashDirective if it fully formed if it's
+  // done the checking already so this is an internal error.
+  assert(getTok().is(AsmToken::Integer) &&
+         "Lexing Cpp line comment: Expected Integer");
   int64_t LineNumber = getTok().getIntVal();
   Lex();
-
-  if (getLexer().isNot(AsmToken::String)) {
-    eatToEndOfLine();
-    return false;
-  }
-
+  assert(getTok().is(AsmToken::String) &&
+         "Lexing Cpp line comment: Expected String");
   StringRef Filename = getTok().getString();
+  Lex();
   // Get rid of the enclosing quotes.
   Filename = Filename.substr(1, Filename.size() - 2);
 
   // Save the SMLoc, Filename and LineNumber for later use by diagnostics.
   CppHashInfo.Loc = L;
   CppHashInfo.Filename = Filename;
   CppHashInfo.LineNumber = LineNumber;
   CppHashInfo.Buf = CurBuffer;
-
-  // Ignore any trailing characters, they're just comment.
-  eatToEndOfLine();
   return false;
 }
 
 /// \brief will use the last parsed cpp hash line filename comment
 /// for the Filename and LineNo if any in the diagnostic.
 void AsmParser::DiagHandler(const SMDiagnostic &Diag, void *Context) {
   const AsmParser *Parser = static_cast<const AsmParser *>(Context);
   raw_ostream &OS = errs();
for (unsigned Parameter = 0; !NParameters || Parameter < NParameters;
     unsigned PI = Parameter;
     if (!FA.Name.empty()) {
       unsigned FAI = 0;
       for (FAI = 0; FAI < NParameters; ++FAI)
         if (M->Parameters[FAI].Name == FA.Name)
           break;
 
       if (FAI >= NParameters) {
         assert(M && "expected macro to be defined");
         Error(IDLoc,
               "parameter named '" + FA.Name + "' does not exist for macro '" +
               M->Name + "'");
         return true;
       }
       PI = FAI;
     }
 
bool AsmParser::parseDirectiveReloc(SMLoc DirectiveLoc) {
   if (!Offset->evaluateAsAbsolute(OffsetValue))
     return Error(OffsetLoc, "expression is not a constant value");
 
   if (OffsetValue < 0)
     return Error(OffsetLoc, "expression is negative");
 
   if (Lexer.isNot(AsmToken::Comma))
     return TokError("expected comma");
-  Lexer.Lex();
+  Lex();
 
   if (Lexer.isNot(AsmToken::Identifier))
     return TokError("expected relocation name");
   SMLoc NameLoc = Lexer.getTok().getLoc();
   StringRef Name = Lexer.getTok().getIdentifier();
-  Lexer.Lex();
+  Lex();
 
   if (Lexer.is(AsmToken::Comma)) {
-    Lexer.Lex();
+    Lex();
     SMLoc ExprLoc = Lexer.getLoc();
     if (parseExpression(Expr))
       return true;
 
     MCValue Value;
     if (!Expr->evaluateAsRelocatable(Value, nullptr, nullptr))
       return Error(ExprLoc, "expression must be relocatable");
   }
while (getLexer().isNot(AsmToken::EndOfStatement)) {
     }
 
     Parameters.push_back(std::move(Parameter));
 
     if (getLexer().is(AsmToken::Comma))
       Lex();
   }
 
-  // Eat the end of statement.
-  Lex();
+  // Eat just the end of statement.
+  Lexer.Lex();
 
   AsmToken EndToken, StartToken = getTok();
   unsigned MacroDepth = 0;
 
   // Lex the macro definition.
   for (;;) {
     // Check whether we have reached the end of the file.
     if (getLexer().is(AsmToken::Eof))
       return Error(DirectiveLoc, "no matching '.endmacro' in definition");
 
     // Otherwise, check whether we have reach the .endmacro.
     if (getLexer().is(AsmToken::Identifier)) {
       if (getTok().getIdentifier() == ".endm" ||
           getTok().getIdentifier() == ".endmacro") {
         if (MacroDepth == 0) { // Outermost macro.
           EndToken = getTok();
-          Lex();
+          Lexer.Lex();
           if (getLexer().isNot(AsmToken::EndOfStatement))
             return TokError("unexpected token in '" + EndToken.getIdentifier() +
                             "' directive");
           break;
         } else {
           // Otherwise we just found the end of an inner macro.
           --MacroDepth;
         }
static bool isSymbolUsedInExpression(const MCSymbol *Sym, const MCExpr *Value) {
   }
 
   llvm_unreachable("Unknown expr kind!");
 }
 
 bool parseAssignmentExpression(StringRef Name, bool allow_redef,
                                MCAsmParser &Parser, MCSymbol *&Sym,
                                const MCExpr *&Value) {
-  MCAsmLexer &Lexer = Parser.getLexer();
 
   // FIXME: Use better location, we should use proper tokens.
-  SMLoc EqualLoc = Lexer.getLoc();
+  SMLoc EqualLoc = Parser.getTok().getLoc();
 
   if (Parser.parseExpression(Value)) {
     Parser.TokError("missing expression");
     Parser.eatToEndOfStatement();
     return true;
   }
 
   // Note: we don't count b as used in "a = b". This is to allow
   // a = b
   // b = c
 
-  if (Lexer.isNot(AsmToken::EndOfStatement))
+  if (Parser.getTok().isNot(AsmToken::EndOfStatement))
     return Parser.TokError("unexpected token in assignment");
 
   // Eat the end of statement marker.
   Parser.Lex();
 
   // Validate that the LHS is allowed to be a variable (either it has not been
   // used as a symbol, or it is an absolute symbol).
   Sym = Parser.getContext().lookupSymbol(Name);

lib/Target/ARM/AsmParser/ARMAsmParser.cpp

bool ARMAsmParser::parseDirectiveObjectArch(SMLoc L) {
   if (getLexer().isNot(AsmToken::Identifier)) {
     Error(getLexer().getLoc(), "unexpected token");
     Parser.eatToEndOfStatement();
     return false;
   }
 
   StringRef Arch = Parser.getTok().getString();
   SMLoc ArchLoc = Parser.getTok().getLoc();
-  getLexer().Lex();
+  Lex();
 
   unsigned ID = ARM::parseArch(Arch);
 
   if (ID == ARM::AK_INVALID) {
     Error(ArchLoc, "unknown architecture '" + Arch + "'");
     Parser.eatToEndOfStatement();
     return false;
   }
bool ARMAsmParser::parseDirectiveArchExtension(SMLoc L) {
   if (getLexer().isNot(AsmToken::Identifier)) {
     Error(getLexer().getLoc(), "unexpected token");
     Parser.eatToEndOfStatement();
     return false;
   }
 
   StringRef Name = Parser.getTok().getString();
   SMLoc ExtLoc = Parser.getTok().getLoc();
-  getLexer().Lex();
+  Lex();
 
   bool EnableFeature = true;
   if (Name.startswith_lower("no")) {
     EnableFeature = false;
     Name = Name.substr(2);
   }
   unsigned FeatureKind = ARM::parseArchExt(Name);
   if (FeatureKind == ARM::AEK_INVALID)

lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp

if (CheckOk) {
     }
   }
 
   return false; // No error
 }
 
 bool HexagonAsmParser::matchBundleOptions() {
   MCAsmParser &Parser = getParser();
-  MCAsmLexer &Lexer = getLexer();
   while (true) {
     if (!Parser.getTok().is(AsmToken::Colon))
       return false;
-    Lexer.Lex();
+    Lex();
     StringRef Option = Parser.getTok().getString();
     if (Option.compare_lower("endloop0") == 0)
       HexagonMCInstrInfo::setInnerLoop(MCB);
     else if (Option.compare_lower("endloop1") == 0)
       HexagonMCInstrInfo::setOuterLoop(MCB);
     else if (Option.compare_lower("mem_noshuf") == 0)
       HexagonMCInstrInfo::setMemReorderDisabled(MCB);
     else if (Option.compare_lower("mem_shuf") == 0)
       HexagonMCInstrInfo::setMemStoreReorderEnabled(MCB);
     else
       return true;
-    Lexer.Lex();
+    Lex();
   }
 }
 
 // For instruction aliases, immediates are generated rather than
 // MCConstantExpr.  Convert them for uniform MCExpr.
 // Also check for signed/unsigned mismatches and warn
 void HexagonAsmParser::canonicalizeImmediates(MCInst &MCI) {
   MCInst NewInst;
return previousEqual(Operands, Index, "loop0") ||
          previousEqual(Operands, Index, "sp3loop0");
 }
 }
 
 bool HexagonAsmParser::splitIdentifier(OperandVector &Operands) {
   AsmToken const &Token = getParser().getTok();
   StringRef String = Token.getString();
   SMLoc Loc = Token.getLoc();
-  getLexer().Lex();
+  Lex();
   do {
     std::pair<StringRef, StringRef> HeadTail = String.split('.');
     if (!HeadTail.first.empty())
       Operands.push_back(HexagonOperand::CreateToken(HeadTail.first, Loc));
     if (!HeadTail.second.empty())
       Operands.push_back(HexagonOperand::CreateToken(
           String.substr(HeadTail.first.size(), 1), Loc));
     String = HeadTail.second;
bool HexagonAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
   bool Again = Lexer.is(AsmToken::Identifier);
   bool NeededWorkaround = false;
   while (Again) {
     AsmToken const &Token = Lexer.getTok();
     RawString = StringRef(RawString.data(),
                           Token.getString().data() - RawString.data () +
                           Token.getString().size());
     Lookahead.push_back(Token);
-    Lexer.Lex();
+    Lex();
     bool Contigious = Lexer.getTok().getString().data() ==
                       Lookahead.back().getString().data() +
                       Lookahead.back().getString().size();
     bool Type = Lexer.is(AsmToken::Identifier) || Lexer.is(AsmToken::Dot) ||
                 Lexer.is(AsmToken::Integer) || Lexer.is(AsmToken::Real) ||
                 Lexer.is(AsmToken::Colon);
     bool Workaround = Lexer.is(AsmToken::Colon) ||
                       Lookahead.back().is(AsmToken::Colon);

 
 bool HexagonAsmParser::parseExpression(MCExpr const *& Expr) {
   llvm::SmallVector<AsmToken, 4> Tokens;
   MCAsmLexer &Lexer = getLexer();
   bool Done = false;
   static char const * Comma = ",";
   do {
     Tokens.emplace_back (Lexer.getTok());
-    Lexer.Lex();
+    Lex();
     switch (Tokens.back().getKind())
     {
     case AsmToken::TokenKind::Hash:
       if (Tokens.size () > 1)
         if ((Tokens.end () - 2)->getKind() == AsmToken::TokenKind::Plus) {
           Tokens.insert(Tokens.end() - 2,
                         AsmToken(AsmToken::TokenKind::Comma, Comma));
           Done = true;

 /// Parse an instruction.
 bool HexagonAsmParser::parseInstruction(OperandVector &Operands) {
   MCAsmParser &Parser = getParser();
   MCAsmLexer &Lexer = getLexer();
   while (true) {
     AsmToken const &Token = Parser.getTok();
     switch (Token.getKind()) {
     case AsmToken::EndOfStatement: {
-      Lexer.Lex();
+      Lex();
       return false;
     }
     case AsmToken::LCurly: {
       if (!Operands.empty())
         return true;
       Operands.push_back(
           HexagonOperand::CreateToken(Token.getString(), Token.getLoc()));
-      Lexer.Lex();
+      Lex();
       return false;
     }
     case AsmToken::RCurly: {
       if (Operands.empty()) {
         Operands.push_back(
             HexagonOperand::CreateToken(Token.getString(), Token.getLoc()));
-        Lexer.Lex();
+        Lex();
       }
       return false;
     }
     case AsmToken::Comma: {
-      Lexer.Lex();
+      Lex();
       continue;
     }
     case AsmToken::EqualEqual:
     case AsmToken::ExclaimEqual:
     case AsmToken::GreaterEqual:
     case AsmToken::GreaterGreater:
     case AsmToken::LessEqual:
     case AsmToken::LessLess: {
       Operands.push_back(HexagonOperand::CreateToken(
           Token.getString().substr(0, 1), Token.getLoc()));
       Operands.push_back(HexagonOperand::CreateToken(
           Token.getString().substr(1, 1), Token.getLoc()));
-      Lexer.Lex();
+      Lex();
       continue;
     }
     case AsmToken::Hash: {
       bool MustNotExtend = false;
       bool ImplicitExpression = implicitExpressionLocation(Operands);
       SMLoc ExprLoc = Lexer.getLoc();
       if (!ImplicitExpression)
         Operands.push_back(
           HexagonOperand::CreateToken(Token.getString(), Token.getLoc()));
-      Lexer.Lex();
+      Lex();
       bool MustExtend = false;
       bool HiOnly = false;
       bool LoOnly = false;
       if (Lexer.is(AsmToken::Hash)) {
-        Lexer.Lex();
+        Lex();
         MustExtend = true;
       } else if (ImplicitExpression)
         MustNotExtend = true;
       AsmToken const &Token = Parser.getTok();
       if (Token.is(AsmToken::Identifier)) {
         StringRef String = Token.getString();
         AsmToken IDToken = Token;
         if (String.lower() == "hi") {
           HiOnly = true;
         } else if (String.lower() == "lo") {
           LoOnly = true;
         }
         if (HiOnly || LoOnly) {
           AsmToken LParen = Lexer.peekTok();
           if (!LParen.is(AsmToken::LParen)) {
             HiOnly = false;
             LoOnly = false;
           } else {
-            Lexer.Lex();
+            Lex();
           }
         }
       }
       MCExpr const *Expr = nullptr;
       if (parseExpression(Expr))
         return true;
       int64_t Value;
       MCContext &Context = Parser.getContext();

lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp

std::unique_ptr<LanaiOperand> LanaiAsmParser::parseIdentifier() {
 
   // If the identifier corresponds to a variant then extract the real
   // identifier.
   if (Kind != LanaiMCExpr::VK_Lanai_None) {
     if (Lexer.getKind() != AsmToken::LParen) {
       Error(Lexer.getLoc(), "Expected '('");
       return 0;
     }
-    Lexer.Lex(); // lex '('
+    Parser.Lex(); // lex '('.
 
     // Parse identifier
     if (Parser.parseIdentifier(Identifier))
       return 0;
   }
 
   // If addition parse the RHS.
   if (Lexer.getKind() == AsmToken::Plus && Parser.parseExpression(RHS))
     return 0;
 
   // For variants parse the final ')'
   if (Kind != LanaiMCExpr::VK_Lanai_None) {
     if (Lexer.getKind() != AsmToken::RParen) {
       Error(Lexer.getLoc(), "Expected ')'");
       return 0;
     }
-    Lexer.Lex(); // lex ')'
+    Lex(); // lex ')'.
   }
 
   End = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
   Res = LanaiMCExpr::create(Kind, Expr, getContext());
 
   // Nest if this was an addition
if (Lexer.is(AsmToken::EndOfStatement) && Name.startswith("bt") &&
       Operands.size() == 3) {
     Operands.erase(Operands.begin(), Operands.begin() + 2);
     Operands.insert(Operands.begin(), LanaiOperand::CreateToken("bt", NameLoc));
   }
 
   // Parse until end of statement, consuming commas between operands
   while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.is(AsmToken::Comma)) {
     // Consume comma token
-    Lexer.Lex();
+    Lex();
 
     // Parse next operand
     if (parseOperand(&Operands, Mnemonic) != MatchOperand_Success)
       return true;
   }
 
   if (IsMemoryAssignmentError(Operands)) {
     Error(Parser.getTok().getLoc(),

lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp

 /// Parse an instruction mnemonic followed by its operands.
 bool PPCAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                                     SMLoc NameLoc, OperandVector &Operands) {
   // The first operand is the token for the instruction name.
   // If the next character is a '+' or '-', we need to add it to the
   // instruction name, to match what TableGen is doing.
   std::string NewOpcode;
   if (getLexer().is(AsmToken::Plus)) {
-    getLexer().Lex();
+    Lex();
     NewOpcode = Name;
     NewOpcode += '+';
     Name = NewOpcode;
   }
   if (getLexer().is(AsmToken::Minus)) {
-    getLexer().Lex();
+    Lex();
     NewOpcode = Name;
     NewOpcode += '-';
     Name = NewOpcode;
   }
   // If the instruction ends in a '.', we need to create a separate
   // token for it, to match what TableGen is doing.
   size_t Dot = Name.find('.');
   StringRef Mnemonic = Name.slice(0, Dot);
bool PPCAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
 
   // Parse the first operand
   if (ParseOperand(Operands))
     return true;
 
   while (getLexer().isNot(AsmToken::EndOfStatement) &&
          getLexer().is(AsmToken::Comma)) {
     // Consume the comma token
-    getLexer().Lex();
+    Lex();
 
     // Parse the next operand
     if (ParseOperand(Operands))
       return true;
   }
 
   // We'll now deal with an unfortunate special case: the syntax for the dcbt
   // and dcbtst instructions differs for server vs. embedded cores.

lib/Target/X86/AsmParser/X86AsmParser.cpp

bool X86AsmParser::ParseIntelIdentifier(const MCExpr *&Val,
   const AsmToken &Tok = Parser.getTok();
   SMLoc Loc = Tok.getLoc();
 
   // Advance the token stream until the end of the current token is
   // after the end of what the frontend claimed.
   const char *EndPtr = Tok.getLoc().getPointer() + LineBuf.size();
   do {
     End = Tok.getEndLoc();
-    getLexer().Lex();
+    getParser().Lex();
   } while (End.getPointer() < EndPtr);
   Identifier = LineBuf;
 
   // The frontend should end parsing on an assembler token boundary, unless it
   // failed parsing.
   assert((End.getPointer() == EndPtr || !Result) &&
          "frontend claimed part of a token?");