Add support for IntegerLiteral in SyntaxTree

[eduucaldas]

Add support for IntegerLiteral in SyntaxTree

[gribozavr2]

clang/include/clang/Tooling/Syntax/Nodes.h

193

"literalToken"?

clang/unittests/Tooling/Syntax/TreeTest.cpp

706 ↗

(On Diff #268382)

Please add more tests for integers with suffixes (0u, 0ll etc.), different bases (hex, octal etc.), digit separators.

http://eel.is/c++draft/lex.icon
https://en.cppreference.com/w/cpp/language/integer_literal

[eduucaldas]

answer comments

[eduucaldas]

fix

[eduucaldas]

answering comments

[gribozavr2]

clang/unittests/Tooling/Syntax/TreeTest.cpp

718 ↗

(On Diff #268420)

Could you remove the whitespace in front of )cpp" for consistency with the rest of the file?

758 ↗

(On Diff #268420)

IntegerLiteralLongLong

792 ↗

(On Diff #268420)

IntegerLiteralBinary

821 ↗

(On Diff #268420)

IntegerLiteralWithDigitSeparators

[gribozavr2]

clang/include/clang/Tooling/Syntax/Nodes.h

645

Could you move this change to a separate patch?

[eduucaldas]

Fix nits

clang/include/clang/Tooling/Syntax/Nodes.h

enum class NodeKind : uint16_t {
   ParametersAndQualifiers,
   MemberPointer
 };
 /// For debugging purposes.
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, NodeKind K);
 
 /// A relation between a parent and child node, e.g. 'left-hand-side of
 /// a binary expression'. Used for implementing accessors.
+// How to name NodeRole:
+// If the child node is a token/keyword, end its name with 'Token'/'Keyword'
+// If this role is not shared between multiple nodes then it is a specific role.
+// In that case prepend with `NodeKind_` of the parent
 enum class NodeRole : uint8_t {
   // Roles common to multiple node kinds.
   /// A node without a parent
   Detached,
   /// Children of an unknown semantic nature, e.g. skipped tokens, comments.
   Unknown,
   /// An opening parenthesis in argument lists and blocks, e.g. '{', '(', etc.
   OpenParen,
   /// A closing parenthesis in argument lists and blocks, e.g. '}', ')', etc.
   CloseParen,
   /// A keywords that introduces some grammar construct, e.g. 'if', 'try', etc.
   IntroducerKeyword,
+  /// A token that represents a literal, e.g. 'nullptr', '1', 'true', etc.
+  LiteralToken,
   /// An inner statement for those that have only a single child of kind
   /// statement, e.g. loop body for while, for, etc; inner statement for case,
   /// default, etc.
   BodyStatement,
 
   // Roles specific to particular node kinds.
-  UnaryOperatorExpression_operatorToken,
+  OperatorExpression_operatorToken,
   UnaryOperatorExpression_operand,
   BinaryOperatorExpression_leftHandSide,
-  BinaryOperatorExpression_operatorToken,
   BinaryOperatorExpression_rightHandSide,
-  CxxNullPtrExpression_keyword,
-  IntegerLiteralExpression_literalToken,
   CaseStatement_value,
   IfStatement_thenStatement,
   IfStatement_elseKeyword,
   IfStatement_elseStatement,
   ReturnStatement_value,
   ExpressionStatement_expression,
   CompoundStatement_statement,
   StaticAssertDeclaration_condition,
   StaticAssertDeclaration_message,
   SimpleDeclaration_declarator,
   TemplateDeclaration_declaration,
   ExplicitTemplateInstantiation_externKeyword,
   ExplicitTemplateInstantiation_declaration,
   ArraySubscript_sizeExpression,
-  TrailingReturnType_arrow,
+  TrailingReturnType_arrowToken,
   TrailingReturnType_declarator,
   ParametersAndQualifiers_parameter,
   ParametersAndQualifiers_trailingReturn
 };
 /// For debugging purposes.
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, NodeRole R);
 
 class SimpleDeclarator;

 
 /// Expression for integer literals.
 class IntegerLiteralExpression final : public Expression {
 public:
   IntegerLiteralExpression() : Expression(NodeKind::IntegerLiteralExpression) {}
   static bool classof(const Node *N) {
     return N->kind() == NodeKind::IntegerLiteralExpression;
   }
   syntax::Leaf *literalToken();
 };
 
 /// An abstract class for prefix and postfix unary operators.
 class UnaryOperatorExpression : public Expression {
 public:
   UnaryOperatorExpression(NodeKind K) : Expression(K) {}
   static bool classof(const Node *N) {
     return N->kind() == NodeKind::PrefixUnaryOperatorExpression ||

 /// E.g. `-> int***`.
 class TrailingReturnType final : public Tree {
 public:
   TrailingReturnType() : Tree(NodeKind::TrailingReturnType) {}
   static bool classof(const Node *N) {
     return N->kind() == NodeKind::TrailingReturnType;
   }
   // TODO: add accessors for specifiers.
   syntax::Leaf *arrowToken();
   syntax::SimpleDeclarator *declarator();
 };
 
 /// Parameter list for a function type and a trailing return type, if the
 /// function has one.
 /// E.g.:
 ///  `(int a) volatile ` in `int foo(int a) volatile;`
 ///  `(int a) &&` in `int foo(int a) &&;`

clang/lib/Tooling/Syntax/BuildTree.cpp

public:
   bool WalkUpFromExpr(Expr *E) {
     assert(!isImplicitExpr(E) && "should be handled by TraverseStmt");
     Builder.foldNode(Builder.getExprRange(E),
                      new (allocator()) syntax::UnknownExpression, E);
     return true;
   }
 
   bool WalkUpFromIntegerLiteral(IntegerLiteral *S) {
-    Builder.markChildToken(
-        S->getLocation(),
-        syntax::NodeRole::IntegerLiteralExpression_literalToken);
+    Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
     Builder.foldNode(Builder.getExprRange(S),
                      new (allocator()) syntax::IntegerLiteralExpression, S);
     return true;
   }
 
   bool WalkUpFromCXXNullPtrLiteralExpr(CXXNullPtrLiteralExpr *S) {
-    Builder.markChildToken(S->getLocation(),
-                           syntax::NodeRole::CxxNullPtrExpression_keyword);
+    Builder.markChildToken(S->getLocation(), syntax::NodeRole::LiteralToken);
     Builder.foldNode(Builder.getExprRange(S),
                      new (allocator()) syntax::CxxNullPtrExpression, S);
     return true;
   }
 
   bool WalkUpFromUnaryOperator(UnaryOperator *S) {
-    Builder.markChildToken(
-        S->getOperatorLoc(),
-        syntax::NodeRole::UnaryOperatorExpression_operatorToken);
+    Builder.markChildToken(S->getOperatorLoc(),
+                           syntax::NodeRole::OperatorExpression_operatorToken);
     Builder.markExprChild(S->getSubExpr(),
                           syntax::NodeRole::UnaryOperatorExpression_operand);
 
     if (S->isPostfix())
       Builder.foldNode(Builder.getExprRange(S),
                        new (allocator()) syntax::PostfixUnaryOperatorExpression,
                        S);
     else
       Builder.foldNode(Builder.getExprRange(S),
                        new (allocator()) syntax::PrefixUnaryOperatorExpression,
                        S);
 
     return true;
   }
 
   bool WalkUpFromBinaryOperator(BinaryOperator *S) {
     Builder.markExprChild(
         S->getLHS(), syntax::NodeRole::BinaryOperatorExpression_leftHandSide);
-    Builder.markChildToken(
-        S->getOperatorLoc(),
-        syntax::NodeRole::BinaryOperatorExpression_operatorToken);
+    Builder.markChildToken(S->getOperatorLoc(),
+                           syntax::NodeRole::OperatorExpression_operatorToken);
     Builder.markExprChild(
         S->getRHS(), syntax::NodeRole::BinaryOperatorExpression_rightHandSide);
     Builder.foldNode(Builder.getExprRange(S),
                      new (allocator()) syntax::BinaryOperatorExpression, S);
     return true;
   }
 
   bool WalkUpFromCXXOperatorCallExpr(CXXOperatorCallExpr *S) {
     if (S->isInfixBinaryOp()) {
       Builder.markExprChild(
           S->getArg(0),
           syntax::NodeRole::BinaryOperatorExpression_leftHandSide);
       Builder.markChildToken(
           S->getOperatorLoc(),
-          syntax::NodeRole::BinaryOperatorExpression_operatorToken);
+          syntax::NodeRole::OperatorExpression_operatorToken);
       Builder.markExprChild(
           S->getArg(1),
           syntax::NodeRole::BinaryOperatorExpression_rightHandSide);
       Builder.foldNode(Builder.getExprRange(S),
                        new (allocator()) syntax::BinaryOperatorExpression, S);
       return true;
     }
     return RecursiveASTVisitor::WalkUpFromCXXOperatorCallExpr(S);
if (ReturnDeclaratorRange.isValid()) {
                        ReturnDeclarator, nullptr);
     }
 
     // Build node for trailing return type.
     auto Return = Builder.getRange(ReturnedType.getSourceRange());
     const auto *Arrow = Return.begin() - 1;
     assert(Arrow->kind() == tok::arrow);
     auto Tokens = llvm::makeArrayRef(Arrow, Return.end());
-    Builder.markChildToken(Arrow, syntax::NodeRole::TrailingReturnType_arrow);
+    Builder.markChildToken(Arrow,
+                           syntax::NodeRole::TrailingReturnType_arrowToken);
     if (ReturnDeclarator)
       Builder.markChild(ReturnDeclarator,
                         syntax::NodeRole::TrailingReturnType_declarator);
     auto *R = new (allocator()) syntax::TrailingReturnType;
     Builder.foldNode(Tokens, R, L);
     return R;
   }
 

clang/lib/Tooling/Syntax/Nodes.cpp

llvm::raw_ostream &syntax::operator<<(llvm::raw_ostream &OS, NodeRole R) {
   case syntax::NodeRole::CaseStatement_value:
     return OS << "CaseStatement_value";
   case syntax::NodeRole::IfStatement_thenStatement:
     return OS << "IfStatement_thenStatement";
   case syntax::NodeRole::IfStatement_elseKeyword:
     return OS << "IfStatement_elseKeyword";
   case syntax::NodeRole::IfStatement_elseStatement:
     return OS << "IfStatement_elseStatement";
-  case syntax::NodeRole::IntegerLiteralExpression_literalToken:
+  case syntax::NodeRole::LiteralToken:
     return OS << "IntegerLiteralExpression_literalToken";
-  case syntax::NodeRole::CxxNullPtrExpression_keyword:
-    return OS << "CxxNullPtrExpression_keyword";
-  case syntax::NodeRole::UnaryOperatorExpression_operatorToken:
-    return OS << "UnaryOperatorExpression_operatorToken";
+  case syntax::NodeRole::OperatorExpression_operatorToken:
+    return OS << "OperatorExpression_operatorToken";
   case syntax::NodeRole::UnaryOperatorExpression_operand:
     return OS << "UnaryOperatorExpression_operand";
   case syntax::NodeRole::BinaryOperatorExpression_leftHandSide:
     return OS << "BinaryOperatorExpression_leftHandSide";
-  case syntax::NodeRole::BinaryOperatorExpression_operatorToken:
-    return OS << "BinaryOperatorExpression_operatorToken";
   case syntax::NodeRole::BinaryOperatorExpression_rightHandSide:
     return OS << "BinaryOperatorExpression_rightHandSide";
   case syntax::NodeRole::ReturnStatement_value:
     return OS << "ReturnStatement_value";
   case syntax::NodeRole::ExpressionStatement_expression:
     return OS << "ExpressionStatement_expression";
   case syntax::NodeRole::CompoundStatement_statement:
     return OS << "CompoundStatement_statement";
   case syntax::NodeRole::StaticAssertDeclaration_condition:
     return OS << "StaticAssertDeclaration_condition";
   case syntax::NodeRole::StaticAssertDeclaration_message:
     return OS << "StaticAssertDeclaration_message";
   case syntax::NodeRole::SimpleDeclaration_declarator:
     return OS << "SimpleDeclaration_declarator";
   case syntax::NodeRole::TemplateDeclaration_declaration:
     return OS << "TemplateDeclaration_declaration";
   case syntax::NodeRole::ExplicitTemplateInstantiation_externKeyword:
     return OS << "ExplicitTemplateInstantiation_externKeyword";
   case syntax::NodeRole::ExplicitTemplateInstantiation_declaration:
     return OS << "ExplicitTemplateInstantiation_declaration";
   case syntax::NodeRole::ArraySubscript_sizeExpression:
     return OS << "ArraySubscript_sizeExpression";
-  case syntax::NodeRole::TrailingReturnType_arrow:
-    return OS << "TrailingReturnType_arrow";
+  case syntax::NodeRole::TrailingReturnType_arrowToken:
+    return OS << "TrailingReturnType_arrowToken";
   case syntax::NodeRole::TrailingReturnType_declarator:
     return OS << "TrailingReturnType_declarator";
   case syntax::NodeRole::ParametersAndQualifiers_parameter:
     return OS << "ParametersAndQualifiers_parameter";
   case syntax::NodeRole::ParametersAndQualifiers_trailingReturn:
     return OS << "ParametersAndQualifiers_trailingReturn";
   }
   llvm_unreachable("invalid role");
 }
 
 syntax::Leaf *syntax::IntegerLiteralExpression::literal() {
   return llvm::cast_or_null<syntax::Leaf>(
-      findChild(syntax::NodeRole::IntegerLiteralExpression_literalToken));
+      findChild(syntax::NodeRole::LiteralToken));
 }
 
 syntax::Leaf *syntax::CxxNullPtrExpression::nullPtrKeyword() {
   return llvm::cast_or_null<syntax::Leaf>(
-      findChild(syntax::NodeRole::CxxNullPtrExpression_keyword));
+      findChild(syntax::NodeRole::LiteralToken));
 }
 
 syntax::Expression *syntax::BinaryOperatorExpression::lhs() {
   return llvm::cast_or_null<syntax::Expression>(
       findChild(syntax::NodeRole::BinaryOperatorExpression_leftHandSide));
 }
 
 syntax::Leaf *syntax::UnaryOperatorExpression::operatorToken() {
   return llvm::cast_or_null<syntax::Leaf>(
-      findChild(syntax::NodeRole::UnaryOperatorExpression_operatorToken));
+      findChild(syntax::NodeRole::OperatorExpression_operatorToken));
 }
 
 syntax::Expression *syntax::UnaryOperatorExpression::operand() {
   return llvm::cast_or_null<syntax::Expression>(
       findChild(syntax::NodeRole::UnaryOperatorExpression_operand));
 }
 
 syntax::Leaf *syntax::BinaryOperatorExpression::operatorToken() {
   return llvm::cast_or_null<syntax::Leaf>(
-      findChild(syntax::NodeRole::BinaryOperatorExpression_operatorToken));
+      findChild(syntax::NodeRole::OperatorExpression_operatorToken));
 }
 
 syntax::Expression *syntax::BinaryOperatorExpression::rhs() {
   return llvm::cast_or_null<syntax::Expression>(
       findChild(syntax::NodeRole::BinaryOperatorExpression_rightHandSide));
 }
 
 syntax::Leaf *syntax::SwitchStatement::switchKeyword() {

 
 syntax::Leaf *syntax::ArraySubscript::rbracket() {
   return llvm::cast_or_null<syntax::Leaf>(
       findChild(syntax::NodeRole::CloseParen));
 }
 
 syntax::Leaf *syntax::TrailingReturnType::arrow() {
   return llvm::cast_or_null<syntax::Leaf>(
-      findChild(syntax::NodeRole::TrailingReturnType_arrow));
+      findChild(syntax::NodeRole::TrailingReturnType_arrowToken));
 }
 
 syntax::SimpleDeclarator *syntax::TrailingReturnType::declarator() {
   return llvm::cast_or_null<syntax::SimpleDeclarator>(
       findChild(syntax::NodeRole::TrailingReturnType_declarator));
 }
 
 syntax::Leaf *syntax::ParametersAndQualifiers::lparen() {