diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h --- a/clang/include/clang/Lex/Token.h +++ b/clang/include/clang/Lex/Token.h @@ -67,6 +67,9 @@ /// Flags - Bits we track about this token, members of the TokenFlags enum. unsigned short Flags; + /// Spelling - How a token is spelt in the source. + const char *Spelling; + public: // Various flags set per token: enum TokenFlags { @@ -237,6 +240,10 @@ Flags |= Flag; } + StringRef getSpelling() const { return {Spelling, getLength()}; } + + void setSpelling(const char *S) { Spelling = S; } + /// Get the specified flag. bool getFlag(TokenFlags Flag) const { return (Flags & Flag) != 0; diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -3264,6 +3264,7 @@ // Read a character, advancing over it. char Char = getAndAdvanceChar(CurPtr, Result); + Result.setSpelling(CurPtr - 1); tok::TokenKind Kind; if (!isVerticalWhitespace(Char)) @@ -3590,12 +3591,14 @@ Char = getCharAndSize(CurPtr, SizeTmp); if (Char == '&') { Kind = tok::ampamp; + // Result.setSpelling(CurPtr - 1); CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); } else if (Char == '=') { Kind = tok::ampequal; CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); } else { Kind = tok::amp; + // Result.setSpelling(CurPtr - 1); } break; case '*': @@ -3639,6 +3642,7 @@ } break; case '~': + // Result.setSpelling(CurPtr - 1); Kind = tok::tilde; break; case '!': @@ -3646,6 +3650,7 @@ Kind = tok::exclaimequal; CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); } else { + // Result.setSpelling(CurPtr - 1); Kind = tok::exclaim; } break; @@ -3846,6 +3851,7 @@ CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); Kind = tok::caretcaret; } else { + // Result.setSpelling(CurPtr - 1); Kind = tok::caret; } break; @@ -3858,9 +3864,11 @@ // If this is '|||||||' and we're in a conflict marker, ignore it. if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) goto LexNextToken; + // Result.setSpelling(CurPtr - 1); Kind = tok::pipepipe; CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); } else { + // Result.setSpelling(CurPtr - 1); Kind = tok::pipe; } break;