diff --git a/mlir/lib/Parser/Lexer.cpp b/mlir/lib/Parser/Lexer.cpp --- a/mlir/lib/Parser/Lexer.cpp +++ b/mlir/lib/Parser/Lexer.cpp @@ -83,8 +83,8 @@ // marker that llvm::MemoryBuffer guarantees will be there. if (curPtr - 1 == curBuffer.end()) return formToken(Token::eof, tokStart); + continue; - LLVM_FALLTHROUGH; case ':': return formToken(Token::colon, tokStart); case ',': diff --git a/mlir/lib/Parser/Token.h b/mlir/lib/Parser/Token.h --- a/mlir/lib/Parser/Token.h +++ b/mlir/lib/Parser/Token.h @@ -23,7 +23,6 @@ #define TOK_IDENTIFIER(NAME) NAME, #define TOK_LITERAL(NAME) NAME, #define TOK_PUNCTUATION(NAME, SPELLING) NAME, -#define TOK_OPERATOR(NAME, SPELLING) NAME, #define TOK_KEYWORD(SPELLING) kw_##SPELLING, #include "TokenKinds.def" }; @@ -50,7 +49,8 @@ bool isNot(Kind k) const { return kind != k; } /// Return true if this token isn't one of the specified kinds. - template bool isNot(Kind k1, Kind k2, T... others) const { + template + bool isNot(Kind k1, Kind k2, T... others) const { return !isAny(k1, k2, others...); } diff --git a/mlir/lib/Parser/Token.cpp b/mlir/lib/Parser/Token.cpp --- a/mlir/lib/Parser/Token.cpp +++ b/mlir/lib/Parser/Token.cpp @@ -145,9 +145,6 @@ #define TOK_PUNCTUATION(NAME, SPELLING) \ case NAME: \ return SPELLING; -#define TOK_OPERATOR(NAME, SPELLING) \ - case NAME: \ - return SPELLING; #define TOK_KEYWORD(SPELLING) \ case kw_##SPELLING: \ return #SPELLING; diff --git a/mlir/lib/Parser/TokenKinds.def b/mlir/lib/Parser/TokenKinds.def --- a/mlir/lib/Parser/TokenKinds.def +++ b/mlir/lib/Parser/TokenKinds.def @@ -11,9 +11,10 @@ // //===----------------------------------------------------------------------===// -#if !defined(TOK_MARKER) && !defined(TOK_IDENTIFIER) && !defined(TOK_LITERAL)&&\ - !defined(TOK_PUNCTUATION) && !defined(TOK_OPERATOR) && !defined(TOK_KEYWORD) -# error Must define one of the TOK_ macros. +#if !defined(TOK_MARKER) && !defined(TOK_IDENTIFIER) && \ + !defined(TOK_LITERAL) && !defined(TOK_PUNCTUATION) && \ + !defined(TOK_KEYWORD) +#error Must define one of the TOK_ macros. #endif #ifndef TOK_MARKER @@ -28,14 +29,10 @@ #ifndef TOK_PUNCTUATION #define TOK_PUNCTUATION(NAME, SPELLING) #endif -#ifndef TOK_OPERATOR -#define TOK_OPERATOR(NAME, SPELLING) -#endif #ifndef TOK_KEYWORD #define TOK_KEYWORD(SPELLING) #endif - // Markers TOK_MARKER(eof) TOK_MARKER(error) @@ -49,34 +46,30 @@ TOK_IDENTIFIER(exclamation_identifier) // !foo // Literals -TOK_LITERAL(floatliteral) // 2.0 -TOK_LITERAL(integer) // 42 -TOK_LITERAL(string) // "foo" -TOK_LITERAL(inttype) // i4, si8, ui16 +TOK_LITERAL(floatliteral) // 2.0 +TOK_LITERAL(integer) // 42 +TOK_LITERAL(string) // "foo" +TOK_LITERAL(inttype) // i4, si8, ui16 // Punctuation. -TOK_PUNCTUATION(arrow, "->") -TOK_PUNCTUATION(at, "@") -TOK_PUNCTUATION(colon, ":") -TOK_PUNCTUATION(comma, ",") -TOK_PUNCTUATION(question, "?") -TOK_PUNCTUATION(l_paren, "(") -TOK_PUNCTUATION(r_paren, ")") -TOK_PUNCTUATION(l_brace, "{") -TOK_PUNCTUATION(r_brace, "}") -TOK_PUNCTUATION(l_square, "[") -TOK_PUNCTUATION(r_square, "]") -TOK_PUNCTUATION(less, "<") -TOK_PUNCTUATION(greater, ">") -TOK_PUNCTUATION(equal, "=") -TOK_PUNCTUATION(ellipsis, "...") -// TODO: More punctuation. - -// Operators. -TOK_OPERATOR(plus, "+") -TOK_OPERATOR(minus, "-") -TOK_OPERATOR(star, "*") -// TODO: More operator tokens +TOK_PUNCTUATION(arrow, "->") +TOK_PUNCTUATION(at, "@") +TOK_PUNCTUATION(colon, ":") +TOK_PUNCTUATION(comma, ",") +TOK_PUNCTUATION(ellipsis, "...") +TOK_PUNCTUATION(equal, "=") +TOK_PUNCTUATION(greater, ">") +TOK_PUNCTUATION(l_brace, "{") +TOK_PUNCTUATION(l_paren, "(") +TOK_PUNCTUATION(l_square, "[") +TOK_PUNCTUATION(less, "<") +TOK_PUNCTUATION(minus, "-") +TOK_PUNCTUATION(plus, "+") +TOK_PUNCTUATION(question, "?") +TOK_PUNCTUATION(r_brace, "}") +TOK_PUNCTUATION(r_paren, ")") +TOK_PUNCTUATION(r_square, "]") +TOK_PUNCTUATION(star, "*") // Keywords. These turn "foo" into Token::kw_foo enums. @@ -122,5 +115,4 @@ #undef TOK_IDENTIFIER #undef TOK_LITERAL #undef TOK_PUNCTUATION -#undef TOK_OPERATOR #undef TOK_KEYWORD diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir --- a/mlir/test/IR/parser.mlir +++ b/mlir/test/IR/parser.mlir @@ -1225,3 +1225,9 @@ return } +// CHECK-LABEL: func @zero_whitespace() { +// CHECK-NEXT: return +func @zero_whitespace() { + // This is a \0 byte. + return +}