clang-format: Print token type name instead of number in -debug output

[djasper]

lib/Format/TokenTypes.def

1

I think I'd prefer to keep these in the same header for now, i.e. declare a macro like.

#define TYPES \

TYPE(ArrayInitializerLSquare) \
TYPE(ArraySubscriptLSquare) \
TYPE(AttributeParen) \
...

and use that instead of #including the .def file.

What do you think?

[djasper]

Oh, and thanks for doing this!

[poiru]

lib/Format/TokenTypes.def

1

Done and committed.

lib/Format/FormatToken.h

 #include "clang/Format/Format.h"
 #include "clang/Lex/Lexer.h"
 #include <memory>
 
 namespace clang {
 namespace format {
 
 enum TokenType {
-  TT_ArrayInitializerLSquare,
-  TT_ArraySubscriptLSquare,
-  TT_AttributeParen,
-  TT_BinaryOperator,
-  TT_BitFieldColon,
-  TT_BlockComment,
-  TT_CastRParen,
-  TT_ConditionalExpr,
-  TT_ConflictAlternative,
-  TT_ConflictEnd,
-  TT_ConflictStart,
-  TT_CtorInitializerColon,
-  TT_CtorInitializerComma,
-  TT_DesignatedInitializerPeriod,
-  TT_DictLiteral,
-  TT_ForEachMacro,
-  TT_FunctionAnnotationRParen,
-  TT_FunctionDeclarationName,
-  TT_FunctionLBrace,
-  TT_FunctionTypeLParen,
-  TT_ImplicitStringLiteral,
-  TT_InheritanceColon,
-  TT_InlineASMBrace,
-  TT_InlineASMColon,
-  TT_JavaAnnotation,
-  TT_JsComputedPropertyName,
-  TT_JsFatArrow,
-  TT_JsTypeColon,
-  TT_JsTypeOptionalQuestion,
-  TT_LambdaArrow,
-  TT_LambdaLSquare,
-  TT_LeadingJavaAnnotation,
-  TT_LineComment,
-  TT_MacroBlockBegin,
-  TT_MacroBlockEnd,
-  TT_ObjCBlockLBrace,
-  TT_ObjCBlockLParen,
-  TT_ObjCDecl,
-  TT_ObjCForIn,
-  TT_ObjCMethodExpr,
-  TT_ObjCMethodSpecifier,
-  TT_ObjCProperty,
-  TT_ObjCStringLiteral,
-  TT_OverloadedOperator,
-  TT_OverloadedOperatorLParen,
-  TT_PointerOrReference,
-  TT_PureVirtualSpecifier,
-  TT_RangeBasedForLoopColon,
-  TT_RegexLiteral,
-  TT_SelectorName,
-  TT_StartOfName,
-  TT_TemplateCloser,
-  TT_TemplateOpener,
-  TT_TemplateString,
-  TT_TrailingAnnotation,
-  TT_TrailingReturnArrow,
-  TT_TrailingUnaryOperator,
-  TT_UnaryOperator,
-  TT_Unknown
+#define TYPE(X) TT_##X,
+#include "TokenTypes.def"
+#undef TYPE
+  NUM_TOKEN_TYPES
 };
 
+/// \brief Determines the name of a token type.
+const char *getTokenTypeName(TokenType Type);
+
 // Represents what type of block a set of braces open.
 enum BraceBlockKind { BK_Unknown, BK_Block, BK_BracedInit };
 
 // The packing kind of a function's parameters.
 enum ParameterPackingKind { PPK_BinPacked, PPK_OnePerLine, PPK_Inconclusive };
 
 enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break };
 

lib/Format/FormatToken.cpp

 #include "clang/Format/Format.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
 #include <climits>
 
 namespace clang {
 namespace format {
 
+const char *getTokenTypeName(TokenType Type) {
+  static const char *const TokNames[] = {
+#define TYPE(X) #X,
+#include "TokenTypes.def"
+#undef TYPE
+    nullptr
+  };
+
+  if (Type < NUM_TOKEN_TYPES)
+    return TokNames[Type];
+  llvm_unreachable("unknown TokenKind");
+  return nullptr;
+}
+
 // FIXME: This is copy&pasted from Sema. Put it in a common place and remove
 // duplication.
 bool FormatToken::isSimpleTypeSpecifier() const {
   switch (Tok.getKind()) {
   case tok::kw_short:
   case tok::kw_long:
   case tok::kw___int64:
   case tok::kw___int128:

lib/Format/TokenAnnotator.cpp

return Left.isOneOf(tok::comma, tok::coloncolon, tok::semi, tok::l_brace,
          (Left.is(tok::l_paren) && !Right.is(tok::r_paren));
 }
 
 void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
   llvm::errs() << "AnnotatedTokens:\n";
   const FormatToken *Tok = Line.First;
   while (Tok) {
     llvm::errs() << " M=" << Tok->MustBreakBefore
-                 << " C=" << Tok->CanBreakBefore << " T=" << Tok->Type
+                 << " C=" << Tok->CanBreakBefore
+                 << " T=" << getTokenTypeName(Tok->Type)
                  << " S=" << Tok->SpacesRequiredBefore
                  << " B=" << Tok->BlockParameterCount
                  << " P=" << Tok->SplitPenalty << " Name=" << Tok->Tok.getName()
                  << " L=" << Tok->TotalLength << " PPK=" << Tok->PackingKind
                  << " FakeLParens=";
     for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
       llvm::errs() << Tok->FakeLParens[i] << "/";
     llvm::errs() << " FakeRParens=" << Tok->FakeRParens << "\n";

lib/Format/TokenTypes.def

+//===--- TokenTypes.def - Token Type Database -------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the TokenType database. Users of this file must
+// optionally #define the TYPE macro to make use of this file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TYPE
+#define TYPE(X)
+#endif
+
+TYPE(ArrayInitializerLSquare)
+TYPE(ArraySubscriptLSquare)
+TYPE(AttributeParen)
+TYPE(BinaryOperator)
+TYPE(BitFieldColon)
+TYPE(BlockComment)
+TYPE(CastRParen)
+TYPE(ConditionalExpr)
+TYPE(ConflictAlternative)
+TYPE(ConflictEnd)
+TYPE(ConflictStart)
+TYPE(CtorInitializerColon)
+TYPE(CtorInitializerComma)
+TYPE(DesignatedInitializerPeriod)
+TYPE(DictLiteral)
+TYPE(ForEachMacro)
+TYPE(FunctionAnnotationRParen)
+TYPE(FunctionDeclarationName)
+TYPE(FunctionLBrace)
+TYPE(FunctionTypeLParen)
+TYPE(ImplicitStringLiteral)
+TYPE(InheritanceColon)
+TYPE(InlineASMBrace)
+TYPE(InlineASMColon)
+TYPE(JavaAnnotation)
+TYPE(JsComputedPropertyName)
+TYPE(JsFatArrow)
+TYPE(JsTypeColon)
+TYPE(JsTypeOptionalQuestion)
+TYPE(LambdaArrow)
+TYPE(LambdaLSquare)
+TYPE(LeadingJavaAnnotation)
+TYPE(LineComment)
+TYPE(MacroBlockBegin)
+TYPE(MacroBlockEnd)
+TYPE(ObjCBlockLBrace)
+TYPE(ObjCBlockLParen)
+TYPE(ObjCDecl)
+TYPE(ObjCForIn)
+TYPE(ObjCMethodExpr)
+TYPE(ObjCMethodSpecifier)
+TYPE(ObjCProperty)
+TYPE(ObjCStringLiteral)
+TYPE(OverloadedOperator)
+TYPE(OverloadedOperatorLParen)
+TYPE(PointerOrReference)
+TYPE(PureVirtualSpecifier)
+TYPE(RangeBasedForLoopColon)
+TYPE(RegexLiteral)
+TYPE(SelectorName)
+TYPE(StartOfName)
+TYPE(TemplateCloser)
+TYPE(TemplateOpener)
+TYPE(TemplateString)
+TYPE(TrailingAnnotation)
+TYPE(TrailingReturnArrow)
+TYPE(TrailingUnaryOperator)
+TYPE(UnaryOperator)
+TYPE(Unknown)