Fix a bug in the .mlir lexer, where a \0 character in a file is treated as a colon (due to an accidental fall through) instead of whitespace.

[lattner]

Summary

While here, simplify the lexer a bit by eliminating the unneeded 'operator'
classification of certain sigils, they can just be treated as 'punctuation'.

[lattner]

Thank you for the review River!

mlir/lib/Parser/Lexer.cpp

case '_':
       // Handle bare identifiers.
       return lexBareIdentifierOrKeyword(tokStart);
 
     case 0:
       // This may either be a nul character in the source file or may be the EOF
       // marker that llvm::MemoryBuffer guarantees will be there.
       if (curPtr - 1 == curBuffer.end())
         return formToken(Token::eof, tokStart);
+      continue;
 
-      LLVM_FALLTHROUGH;
     case ':':
       return formToken(Token::colon, tokStart);
     case ',':
       return formToken(Token::comma, tokStart);
     case '.':
       return lexEllipsis(tokStart);
     case '(':
       return formToken(Token::l_paren, tokStart);

mlir/lib/Parser/Token.h

 /// This represents a token in the MLIR syntax.
 class Token {
 public:
   enum Kind {
 #define TOK_MARKER(NAME) NAME,
 #define TOK_IDENTIFIER(NAME) NAME,
 #define TOK_LITERAL(NAME) NAME,
 #define TOK_PUNCTUATION(NAME, SPELLING) NAME,
-#define TOK_OPERATOR(NAME, SPELLING) NAME,
 #define TOK_KEYWORD(SPELLING) kw_##SPELLING,
 #include "TokenKinds.def"
   };
 
   Token(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}
 
   // Return the bytes that make up this token.
   StringRef getSpelling() const { return spelling; }
bool isAny(Kind k1, Kind k2, Kind k3, T... others) const {
     if (is(k1))
       return true;
     return isAny(k2, k3, others...);
   }
 
   bool isNot(Kind k) const { return kind != k; }
 
   /// Return true if this token isn't one of the specified kinds.
-  template <typename... T> bool isNot(Kind k1, Kind k2, T... others) const {
+  template <typename... T>
+  bool isNot(Kind k1, Kind k2, T... others) const {
     return !isAny(k1, k2, others...);
   }
 
   /// Return true if this is one of the keyword token kinds (e.g. kw_if).
   bool isKeyword() const;
 
   // Helpers to decode specific sorts of tokens.
 

mlir/lib/Parser/Token.cpp

 /// literal tokens since they have no fixed spelling.
 StringRef Token::getTokenSpelling(Kind kind) {
   switch (kind) {
   default:
     llvm_unreachable("This token kind has no fixed spelling");
 #define TOK_PUNCTUATION(NAME, SPELLING)                                        \
   case NAME:                                                                   \
     return SPELLING;
-#define TOK_OPERATOR(NAME, SPELLING)                                           \
-  case NAME:                                                                   \
-    return SPELLING;
 #define TOK_KEYWORD(SPELLING)                                                  \
   case kw_##SPELLING:                                                          \
     return #SPELLING;
 #include "TokenKinds.def"
   }
 }
 
 /// Return true if this is one of the keyword token kinds (e.g. kw_if).

mlir/lib/Parser/TokenKinds.def

 //===- TokenKinds.def - MLIR Token Description ------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file is intended to be #include'd multiple times to extract information
 // about tokens for various clients in the lexer.
 //
 //===----------------------------------------------------------------------===//
 
-#if !defined(TOK_MARKER) && !defined(TOK_IDENTIFIER) && !defined(TOK_LITERAL)&&\
-    !defined(TOK_PUNCTUATION) && !defined(TOK_OPERATOR) && !defined(TOK_KEYWORD)
-#  error Must define one of the TOK_ macros.
+#if !defined(TOK_MARKER) && !defined(TOK_IDENTIFIER) &&                        \
+    !defined(TOK_LITERAL) && !defined(TOK_PUNCTUATION) &&                      \
+    !defined(TOK_KEYWORD)
+#error Must define one of the TOK_ macros.
 #endif
 
 #ifndef TOK_MARKER
 #define TOK_MARKER(X)
 #endif
 #ifndef TOK_IDENTIFIER
 #define TOK_IDENTIFIER(NAME)
 #endif
 #ifndef TOK_LITERAL
 #define TOK_LITERAL(NAME)
 #endif
 #ifndef TOK_PUNCTUATION
 #define TOK_PUNCTUATION(NAME, SPELLING)
 #endif
-#ifndef TOK_OPERATOR
-#define TOK_OPERATOR(NAME, SPELLING)
-#endif
 #ifndef TOK_KEYWORD
 #define TOK_KEYWORD(SPELLING)
 #endif
 
-
 // Markers
 TOK_MARKER(eof)
 TOK_MARKER(error)
 
 // Identifiers.
 TOK_IDENTIFIER(bare_identifier)        // foo
 TOK_IDENTIFIER(at_identifier)          // @foo
 TOK_IDENTIFIER(hash_identifier)        // #foo
 TOK_IDENTIFIER(percent_identifier)     // %foo
 TOK_IDENTIFIER(caret_identifier)       // ^foo
 TOK_IDENTIFIER(exclamation_identifier) // !foo
 
 // Literals
-TOK_LITERAL(floatliteral)               // 2.0
-TOK_LITERAL(integer)                    // 42
-TOK_LITERAL(string)                     // "foo"
-TOK_LITERAL(inttype)                    // i4, si8, ui16
+TOK_LITERAL(floatliteral) // 2.0
+TOK_LITERAL(integer)      // 42
+TOK_LITERAL(string)       // "foo"
+TOK_LITERAL(inttype)      // i4, si8, ui16
 
 // Punctuation.
-TOK_PUNCTUATION(arrow,            "->")
-TOK_PUNCTUATION(at,               "@")
-TOK_PUNCTUATION(colon,            ":")
-TOK_PUNCTUATION(comma,            ",")
-TOK_PUNCTUATION(question,         "?")
-TOK_PUNCTUATION(l_paren,          "(")
-TOK_PUNCTUATION(r_paren,          ")")
-TOK_PUNCTUATION(l_brace,          "{")
-TOK_PUNCTUATION(r_brace,          "}")
-TOK_PUNCTUATION(l_square,         "[")
-TOK_PUNCTUATION(r_square,         "]")
-TOK_PUNCTUATION(less,             "<")
-TOK_PUNCTUATION(greater,          ">")
-TOK_PUNCTUATION(equal,            "=")
-TOK_PUNCTUATION(ellipsis,         "...")
-// TODO: More punctuation.
-
-// Operators.
-TOK_OPERATOR(plus,               "+")
-TOK_OPERATOR(minus,              "-")
-TOK_OPERATOR(star,               "*")
-// TODO: More operator tokens
+TOK_PUNCTUATION(arrow, "->")
+TOK_PUNCTUATION(at, "@")
+TOK_PUNCTUATION(colon, ":")
+TOK_PUNCTUATION(comma, ",")
+TOK_PUNCTUATION(ellipsis, "...")
+TOK_PUNCTUATION(equal, "=")
+TOK_PUNCTUATION(greater, ">")
+TOK_PUNCTUATION(l_brace, "{")
+TOK_PUNCTUATION(l_paren, "(")
+TOK_PUNCTUATION(l_square, "[")
+TOK_PUNCTUATION(less, "<")
+TOK_PUNCTUATION(minus, "-")
+TOK_PUNCTUATION(plus, "+")
+TOK_PUNCTUATION(question, "?")
+TOK_PUNCTUATION(r_brace, "}")
+TOK_PUNCTUATION(r_paren, ")")
+TOK_PUNCTUATION(r_square, "]")
+TOK_PUNCTUATION(star, "*")
 
 // Keywords.  These turn "foo" into Token::kw_foo enums.
 
 // NOTE: Please key these alphabetized to make it easier to find something in
 // this list and to cater to OCD.
 TOK_KEYWORD(affine_map)
 TOK_KEYWORD(affine_set)
 TOK_KEYWORD(attributes)

 TOK_KEYWORD(type)
 TOK_KEYWORD(unit)
 TOK_KEYWORD(vector)
 
 #undef TOK_MARKER
 #undef TOK_IDENTIFIER
 #undef TOK_LITERAL
 #undef TOK_PUNCTUATION
-#undef TOK_OPERATOR
 #undef TOK_KEYWORD

mlir/test/IR/parser.mlir

func @pretty_names() {
 
   %q:3, %r = test.string_attr_pretty_name
   // CHECK: %q, %q_1, %q_2, %r = test.string_attr_pretty_name attributes {names = ["q", "q", "q", "r"]}
 
   // CHECK: return
   return
 }
 
+// CHECK-LABEL: func @zero_whitespace() {
+// CHECK-NEXT: return
+func @zero_whitespace() {
+  ␀  // This is a \0 byte.
+  return
+}