clang-format: Print token type name instead of number in -debug output

[djasper]

lib/Format/TokenTypes.def

1 ↗

(On Diff #29523)

I think I'd prefer to keep these in the same header for now, i.e. declare a macro like.

#define TYPES \

TYPE(ArrayInitializerLSquare) \
TYPE(ArraySubscriptLSquare) \
TYPE(AttributeParen) \
...

and use that instead of #including the .def file.

What do you think?

[djasper]

Oh, and thanks for doing this!

[poiru]

lib/Format/TokenTypes.def

1 ↗

(On Diff #29523)

Done and committed.

cfe/trunk/lib/Format/FormatToken.h

 #include "clang/Basic/OperatorPrecedence.h"
 #include "clang/Format/Format.h"
 #include "clang/Lex/Lexer.h"
 #include <memory>
 
 namespace clang {
 namespace format {
 
+#define LIST_TOKEN_TYPES \
+  TYPE(ArrayInitializerLSquare) \
+  TYPE(ArraySubscriptLSquare) \
+  TYPE(AttributeParen) \
+  TYPE(BinaryOperator) \
+  TYPE(BitFieldColon) \
+  TYPE(BlockComment) \
+  TYPE(CastRParen) \
+  TYPE(ConditionalExpr) \
+  TYPE(ConflictAlternative) \
+  TYPE(ConflictEnd) \
+  TYPE(ConflictStart) \
+  TYPE(CtorInitializerColon) \
+  TYPE(CtorInitializerComma) \
+  TYPE(DesignatedInitializerPeriod) \
+  TYPE(DictLiteral) \
+  TYPE(ForEachMacro) \
+  TYPE(FunctionAnnotationRParen) \
+  TYPE(FunctionDeclarationName) \
+  TYPE(FunctionLBrace) \
+  TYPE(FunctionTypeLParen) \
+  TYPE(ImplicitStringLiteral) \
+  TYPE(InheritanceColon) \
+  TYPE(InlineASMBrace) \
+  TYPE(InlineASMColon) \
+  TYPE(JavaAnnotation) \
+  TYPE(JsComputedPropertyName) \
+  TYPE(JsFatArrow) \
+  TYPE(JsTypeColon) \
+  TYPE(JsTypeOptionalQuestion) \
+  TYPE(LambdaArrow) \
+  TYPE(LambdaLSquare) \
+  TYPE(LeadingJavaAnnotation) \
+  TYPE(LineComment) \
+  TYPE(MacroBlockBegin) \
+  TYPE(MacroBlockEnd) \
+  TYPE(ObjCBlockLBrace) \
+  TYPE(ObjCBlockLParen) \
+  TYPE(ObjCDecl) \
+  TYPE(ObjCForIn) \
+  TYPE(ObjCMethodExpr) \
+  TYPE(ObjCMethodSpecifier) \
+  TYPE(ObjCProperty) \
+  TYPE(ObjCStringLiteral) \
+  TYPE(OverloadedOperator) \
+  TYPE(OverloadedOperatorLParen) \
+  TYPE(PointerOrReference) \
+  TYPE(PureVirtualSpecifier) \
+  TYPE(RangeBasedForLoopColon) \
+  TYPE(RegexLiteral) \
+  TYPE(SelectorName) \
+  TYPE(StartOfName) \
+  TYPE(TemplateCloser) \
+  TYPE(TemplateOpener) \
+  TYPE(TemplateString) \
+  TYPE(TrailingAnnotation) \
+  TYPE(TrailingReturnArrow) \
+  TYPE(TrailingUnaryOperator) \
+  TYPE(UnaryOperator) \
+  TYPE(Unknown)
+
 enum TokenType {
-  TT_ArrayInitializerLSquare,
-  TT_ArraySubscriptLSquare,
-  TT_AttributeParen,
-  TT_BinaryOperator,
-  TT_BitFieldColon,
-  TT_BlockComment,
-  TT_CastRParen,
-  TT_ConditionalExpr,
-  TT_ConflictAlternative,
-  TT_ConflictEnd,
-  TT_ConflictStart,
-  TT_CtorInitializerColon,
-  TT_CtorInitializerComma,
-  TT_DesignatedInitializerPeriod,
-  TT_DictLiteral,
-  TT_ForEachMacro,
-  TT_FunctionAnnotationRParen,
-  TT_FunctionDeclarationName,
-  TT_FunctionLBrace,
-  TT_FunctionTypeLParen,
-  TT_ImplicitStringLiteral,
-  TT_InheritanceColon,
-  TT_InlineASMBrace,
-  TT_InlineASMColon,
-  TT_JavaAnnotation,
-  TT_JsComputedPropertyName,
-  TT_JsFatArrow,
-  TT_JsTypeColon,
-  TT_JsTypeOptionalQuestion,
-  TT_LambdaArrow,
-  TT_LambdaLSquare,
-  TT_LeadingJavaAnnotation,
-  TT_LineComment,
-  TT_MacroBlockBegin,
-  TT_MacroBlockEnd,
-  TT_ObjCBlockLBrace,
-  TT_ObjCBlockLParen,
-  TT_ObjCDecl,
-  TT_ObjCForIn,
-  TT_ObjCMethodExpr,
-  TT_ObjCMethodSpecifier,
-  TT_ObjCProperty,
-  TT_ObjCStringLiteral,
-  TT_OverloadedOperator,
-  TT_OverloadedOperatorLParen,
-  TT_PointerOrReference,
-  TT_PureVirtualSpecifier,
-  TT_RangeBasedForLoopColon,
-  TT_RegexLiteral,
-  TT_SelectorName,
-  TT_StartOfName,
-  TT_TemplateCloser,
-  TT_TemplateOpener,
-  TT_TemplateString,
-  TT_TrailingAnnotation,
-  TT_TrailingReturnArrow,
-  TT_TrailingUnaryOperator,
-  TT_UnaryOperator,
-  TT_Unknown
+#define TYPE(X) TT_##X,
+LIST_TOKEN_TYPES
+#undef TYPE
+  NUM_TOKEN_TYPES
 };
 
+/// \brief Determines the name of a token type.
+const char *getTokenTypeName(TokenType Type);
+
 // Represents what type of block a set of braces open.
 enum BraceBlockKind { BK_Unknown, BK_Block, BK_BracedInit };
 
 // The packing kind of a function's parameters.
 enum ParameterPackingKind { PPK_BinPacked, PPK_OnePerLine, PPK_Inconclusive };
 
 enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break };
 

cfe/trunk/lib/Format/FormatToken.cpp

 #include "clang/Format/Format.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
 #include <climits>
 
 namespace clang {
 namespace format {
 
+const char *getTokenTypeName(TokenType Type) {
+  static const char *const TokNames[] = {
+#define TYPE(X) #X,
+LIST_TOKEN_TYPES
+#undef TYPE
+    nullptr
+  };
+
+  if (Type < NUM_TOKEN_TYPES)
+    return TokNames[Type];
+  llvm_unreachable("unknown TokenType");
+  return nullptr;
+}
+
 // FIXME: This is copy&pasted from Sema. Put it in a common place and remove
 // duplication.
 bool FormatToken::isSimpleTypeSpecifier() const {
   switch (Tok.getKind()) {
   case tok::kw_short:
   case tok::kw_long:
   case tok::kw___int64:
   case tok::kw___int128:

cfe/trunk/lib/Format/TokenAnnotator.cpp

return Left.isOneOf(tok::comma, tok::coloncolon, tok::semi, tok::l_brace,
          (Left.is(tok::l_paren) && !Right.is(tok::r_paren));
 }
 
 void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
   llvm::errs() << "AnnotatedTokens:\n";
   const FormatToken *Tok = Line.First;
   while (Tok) {
     llvm::errs() << " M=" << Tok->MustBreakBefore
-                 << " C=" << Tok->CanBreakBefore << " T=" << Tok->Type
+                 << " C=" << Tok->CanBreakBefore
+                 << " T=" << getTokenTypeName(Tok->Type)
                  << " S=" << Tok->SpacesRequiredBefore
                  << " B=" << Tok->BlockParameterCount
                  << " P=" << Tok->SplitPenalty << " Name=" << Tok->Tok.getName()
                  << " L=" << Tok->TotalLength << " PPK=" << Tok->PackingKind
                  << " FakeLParens=";
     for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
       llvm::errs() << Tok->FakeLParens[i] << "/";
     llvm::errs() << " FakeRParens=" << Tok->FakeRParens << "\n";