[clang-format] Rename common types between C#/JS

[curdeius]

LGTM pending CI.

[exv]

Rebase

[exv]

Rebase again in an attempt to fix CI

clang/lib/Format/FormatToken.h

#define LIST_TOKEN_TYPES                                                       \
   TYPE(ConflictEnd)                                                            \
   TYPE(ConflictStart)                                                          \
   TYPE(ConstraintJunctions)                                                    \
   TYPE(CtorInitializerColon)                                                   \
   TYPE(CtorInitializerComma)                                                   \
   TYPE(DesignatedInitializerLSquare)                                           \
   TYPE(DesignatedInitializerPeriod)                                            \
   TYPE(DictLiteral)                                                            \
+  TYPE(FatArrow)                                                               \
   TYPE(ForEachMacro)                                                           \
   TYPE(FunctionAnnotationRParen)                                               \
   TYPE(FunctionDeclarationName)                                                \
   TYPE(FunctionLBrace)                                                         \
   TYPE(FunctionTypeLParen)                                                     \
   TYPE(ImplicitStringLiteral)                                                  \
   TYPE(InheritanceColon)                                                       \
   TYPE(InheritanceComma)                                                       \
   TYPE(InlineASMBrace)                                                         \
   TYPE(InlineASMColon)                                                         \
   TYPE(InlineASMSymbolicNameLSquare)                                           \
   TYPE(JavaAnnotation)                                                         \
   TYPE(JsComputedPropertyName)                                                 \
   TYPE(JsExponentiation)                                                       \
   TYPE(JsExponentiationEqual)                                                  \
-  TYPE(JsFatArrow)                                                             \
-  TYPE(JsNonNullAssertion)                                                     \
-  TYPE(JsNullishCoalescingOperator)                                            \
-  TYPE(JsNullPropagatingOperator)                                              \
+  TYPE(JsPipePipeEqual)                                                        \
   TYPE(JsPrivateIdentifier)                                                    \
   TYPE(JsTypeColon)                                                            \
   TYPE(JsTypeOperator)                                                         \
   TYPE(JsTypeOptionalQuestion)                                                 \
   TYPE(JsAndAndEqual)                                                          \
-  TYPE(JsPipePipeEqual)                                                        \
-  TYPE(JsNullishCoalescingEqual)                                               \
   TYPE(LambdaArrow)                                                            \
   TYPE(LambdaLBrace)                                                           \
   TYPE(LambdaLSquare)                                                          \
   TYPE(LeadingJavaAnnotation)                                                  \
   TYPE(LineComment)                                                            \
   TYPE(MacroBlockBegin)                                                        \
   TYPE(MacroBlockEnd)                                                          \
   TYPE(NamespaceMacro)                                                         \
+  TYPE(NonNullAssertion)                                                       \
+  TYPE(NullCoalescingEqual)                                                    \
+  TYPE(NullCoalescingOperator)                                                 \
+  TYPE(NullPropagatingOperator)                                                \
   TYPE(ObjCBlockLBrace)                                                        \
   TYPE(ObjCBlockLParen)                                                        \
   TYPE(ObjCDecl)                                                               \
   TYPE(ObjCForIn)                                                              \
   TYPE(ObjCMethodExpr)                                                         \
   TYPE(ObjCMethodSpecifier)                                                    \
   TYPE(ObjCProperty)                                                           \
   TYPE(ObjCStringLiteral)                                                      \

clang/lib/Format/FormatTokenLexer.cpp

void FormatTokenLexer::tryMergePreviousTokens() {
 
   if (Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) {
     static const tok::TokenKind NullishCoalescingOperator[] = {tok::question,
                                                                tok::question};
     static const tok::TokenKind NullPropagatingOperator[] = {tok::question,
                                                              tok::period};
     static const tok::TokenKind FatArrow[] = {tok::equal, tok::greater};
 
-    if (tryMergeTokens(FatArrow, TT_JsFatArrow))
+    if (tryMergeTokens(FatArrow, TT_FatArrow))
       return;
-    if (tryMergeTokens(NullishCoalescingOperator,
-                       TT_JsNullishCoalescingOperator)) {
+    if (tryMergeTokens(NullishCoalescingOperator, TT_NullCoalescingOperator)) {
       // Treat like the "||" operator (as opposed to the ternary ?).
       Tokens.back()->Tok.setKind(tok::pipepipe);
       return;
     }
-    if (tryMergeTokens(NullPropagatingOperator, TT_JsNullPropagatingOperator)) {
+    if (tryMergeTokens(NullPropagatingOperator, TT_NullPropagatingOperator)) {
       // Treat like a regular "." access.
       Tokens.back()->Tok.setKind(tok::period);
       return;
     }
     if (tryMergeNullishCoalescingEqual()) {
       return;
     }
   }
const llvm::StringSet<> FormatTokenLexer::CSharpAttributeTargets = {
     "param",    "property", "return", "type",
 };
 
 bool FormatTokenLexer::tryMergeNullishCoalescingEqual() {
   if (Tokens.size() < 2)
     return false;
   auto &NullishCoalescing = *(Tokens.end() - 2);
   auto &Equal = *(Tokens.end() - 1);
-  if (NullishCoalescing->getType() != TT_JsNullishCoalescingOperator ||
+  if (NullishCoalescing->getType() != TT_NullCoalescingOperator ||
       !Equal->is(tok::equal))
     return false;
   NullishCoalescing->Tok.setKind(tok::equal); // no '??=' in clang tokens.
   NullishCoalescing->TokenText =
       StringRef(NullishCoalescing->TokenText.begin(),
                 Equal->TokenText.end() - NullishCoalescing->TokenText.begin());
   NullishCoalescing->ColumnWidth += Equal->ColumnWidth;
-  NullishCoalescing->setType(TT_JsNullishCoalescingEqual);
+  NullishCoalescing->setType(TT_NullCoalescingEqual);
   Tokens.erase(Tokens.end() - 1);
   return true;
 }
 
 bool FormatTokenLexer::tryMergeCSharpKeywordVariables() {
   if (Tokens.size() < 2)
     return false;
   auto &At = *(Tokens.end() - 2);

clang/lib/Format/TokenAnnotator.cpp

void resetTokenMetadata(FormatToken *Token) {
     if (!Token)
       return;
 
     // Reset token type in case we have already looked at it and then
     // recovered from an error (e.g. failure to find the matching >).
     if (!CurrentToken->isOneOf(
             TT_LambdaLSquare, TT_LambdaLBrace, TT_AttributeMacro,
             TT_ForEachMacro, TT_TypenameMacro, TT_FunctionLBrace,
-            TT_ImplicitStringLiteral, TT_InlineASMBrace, TT_JsFatArrow,
+            TT_ImplicitStringLiteral, TT_InlineASMBrace, TT_FatArrow,
             TT_LambdaArrow, TT_NamespaceMacro, TT_OverloadedOperator,
             TT_RegexLiteral, TT_TemplateString, TT_ObjCStringLiteral,
             TT_UntouchableMacroFunc, TT_ConstraintJunctions,
             TT_StatementAttributeLikeMacro))
       CurrentToken->setType(TT_Unknown);
     CurrentToken->Role.reset();
     CurrentToken->MatchingParen = nullptr;
     CurrentToken->FakeLParens.clear();
if ((Style.Language == FormatStyle::LK_JavaScript || Style.isCSharp()) &&
                       *Current.Previous, /* AcceptIdentifierName= */ true)
                 : Current.Previous->is(tok::identifier);
         if (IsIdentifier ||
             Current.Previous->isOneOf(
                 tok::kw_namespace, tok::r_paren, tok::r_square, tok::r_brace,
                 tok::kw_false, tok::kw_true, Keywords.kw_type, Keywords.kw_get,
                 Keywords.kw_set) ||
             Current.Previous->Tok.isLiteral()) {
-          Current.setType(TT_JsNonNullAssertion);
+          Current.setType(TT_NonNullAssertion);
           return;
         }
       }
       if (Current.Next &&
           Current.Next->isOneOf(TT_BinaryOperator, Keywords.kw_as)) {
-        Current.setType(TT_JsNonNullAssertion);
+        Current.setType(TT_NonNullAssertion);
         return;
       }
     }
 
     // Line.MightBeFunctionDecl can only be true after the parentheses of a
     // function declaration have been found. In this case, 'Current' is a
     // trailing token of this declaration and thus cannot be a name.
     if (Current.is(Keywords.kw_instanceof)) {
if (Current) {
            ((Style.Language == FormatStyle::LK_Proto ||
              Style.Language == FormatStyle::LK_TextProto) &&
             NextNonComment->is(tok::less))))
         return prec::Assignment;
       if (Current->is(TT_JsComputedPropertyName))
         return prec::Assignment;
       if (Current->is(TT_LambdaArrow))
         return prec::Comma;
-      if (Current->is(TT_JsFatArrow))
+      if (Current->is(TT_FatArrow))
         return prec::Assignment;
       if (Current->isOneOf(tok::semi, TT_InlineASMColon, TT_SelectorName) ||
           (Current->is(tok::comment) && NextNonComment &&
            NextNonComment->is(TT_SelectorName)))
         return 0;
       if (Current->is(TT_RangeBasedForLoopColon))
         return prec::Comma;
       if ((Style.Language == FormatStyle::LK_Java ||
if (Style.isCpp()) {
     // Spaces inside braces.
     if (Left.is(tok::l_brace) && Right.isNot(tok::r_brace))
       return true;
 
     if (Left.isNot(tok::l_brace) && Right.is(tok::r_brace))
       return true;
 
     // Spaces around '=>'.
-    if (Left.is(TT_JsFatArrow) || Right.is(TT_JsFatArrow))
+    if (Left.is(TT_FatArrow) || Right.is(TT_FatArrow))
       return true;
 
     // No spaces around attribute target colons
     if (Left.is(TT_AttributeColon) || Right.is(TT_AttributeColon))
       return false;
 
     // space between type and variable e.g. Dictionary<string,string> foo;
     if (Left.is(TT_TemplateCloser) && Right.is(TT_StartOfName))
       return true;
 
     // spaces inside square brackets.
     if (Left.is(tok::l_square) || Right.is(tok::r_square))
       return Style.SpacesInSquareBrackets;
 
     // No space before ? in nullable types.
     if (Right.is(TT_CSharpNullable))
       return false;
 
     // No space before null forgiving '!'.
-    if (Right.is(TT_JsNonNullAssertion))
+    if (Right.is(TT_NonNullAssertion))
       return false;
 
     // No space between consecutive commas '[,,]'.
     if (Left.is(tok::comma) && Right.is(tok::comma))
       return false;
 
     // space after var in `var (key, value)`
     if (Left.is(Keywords.kw_var) && Right.is(tok::l_paren))
if (Style.isCpp()) {
     if (Left.isOneOf(tok::kw_public, tok::kw_private, tok::kw_protected,
                      tok::kw_virtual, tok::kw_extern, tok::kw_static,
                      Keywords.kw_internal, Keywords.kw_abstract,
                      Keywords.kw_sealed, Keywords.kw_override,
                      Keywords.kw_async, Keywords.kw_unsafe) &&
         Right.is(tok::l_paren))
       return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
-    if (Left.is(TT_JsFatArrow))
+    if (Left.is(TT_FatArrow))
       return true;
     // for await ( ...
     if (Right.is(tok::l_paren) && Left.is(Keywords.kw_await) && Left.Previous &&
         Left.Previous->is(tok::kw_for))
       return true;
     if (Left.is(Keywords.kw_async) && Right.is(tok::l_paren) &&
         Right.MatchingParen) {
       const FormatToken *Next = Right.MatchingParen->getNextNonComment();
       // An async arrow function, for example: `x = async () => foo();`,
       // as opposed to calling a function called async: `x = async();`
-      if (Next && Next->is(TT_JsFatArrow))
+      if (Next && Next->is(TT_FatArrow))
         return true;
     }
     if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
         (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
       return false;
     // In tagged template literals ("html`bar baz`"), there is no space between
     // the tag identifier and the template string.
     if (Keywords.IsJavaScriptIdentifier(Left,
if (Left.is(tok::ellipsis))
       return false;
     if (Left.is(TT_TemplateCloser) &&
         !Right.isOneOf(tok::equal, tok::l_brace, tok::comma, tok::l_square,
                        Keywords.kw_implements, Keywords.kw_extends))
       // Type assertions ('<type>expr') are not followed by whitespace. Other
       // locations that should have whitespace following are identified by the
       // above set of follower tokens.
       return false;
-    if (Right.is(TT_JsNonNullAssertion))
+    if (Right.is(TT_NonNullAssertion))
       return false;
-    if (Left.is(TT_JsNonNullAssertion) &&
+    if (Left.is(TT_NonNullAssertion) &&
         Right.isOneOf(Keywords.kw_as, Keywords.kw_in))
       return true; // "x! as string", "x! in y"
   } else if (Style.Language == FormatStyle::LK_Java) {
     if (Left.is(tok::r_square) && Right.is(tok::l_brace))
       return true;
     if (Left.is(Keywords.kw_synchronized) && Right.is(tok::l_paren))
       return Style.SpaceBeforeParens != FormatStyle::SBPO_Never;
     if ((Left.isOneOf(tok::kw_static, tok::kw_public, tok::kw_private,
if (Left.is(tok::l_brace) && Line.Level == 0 &&
         (Line.startsWith(tok::kw_enum) ||
          Line.startsWith(tok::kw_const, tok::kw_enum) ||
          Line.startsWith(tok::kw_export, tok::kw_enum) ||
          Line.startsWith(tok::kw_export, tok::kw_const, tok::kw_enum)))
       // JavaScript top-level enum key/value pairs are put on separate lines
       // instead of bin-packing.
       return true;
     if (Right.is(tok::r_brace) && Left.is(tok::l_brace) && Left.Previous &&
-        Left.Previous->is(TT_JsFatArrow)) {
+        Left.Previous->is(TT_FatArrow)) {
       // JS arrow function (=> {...}).
       switch (Style.AllowShortLambdasOnASingleLine) {
       case FormatStyle::SLS_All:
         return false;
       case FormatStyle::SLS_None:
         return true;
       case FormatStyle::SLS_Empty:
         return !Left.Children.empty();
if (Style.isCSharp()) {
     if (Right.NestingLevel == 0 &&
         (Left.Tok.getIdentifierInfo() ||
          Left.isOneOf(tok::r_square, tok::r_paren)) &&
         Right.isOneOf(tok::l_square, tok::l_paren))
       return false; // Otherwise automatic semicolon insertion would trigger.
     if (NonComment && NonComment->is(tok::identifier) &&
         NonComment->TokenText == "asserts")
       return false;
-    if (Left.is(TT_JsFatArrow) && Right.is(tok::l_brace))
+    if (Left.is(TT_FatArrow) && Right.is(tok::l_brace))
       return false;
     if (Left.is(TT_JsTypeColon))
       return true;
     // Don't wrap between ":" and "!" of a strict prop init ("field!: type;").
     if (Left.is(tok::exclaim) && Right.is(tok::colon))
       return false;
     // Look for is type annotations like:
     // function f(): a is B { ... }
if (Style.isCSharp()) {
     if (Right.isOneOf(Keywords.kw_extends, Keywords.kw_infer)) {
       // extends and infer can appear as keywords in conditional types:
       //   https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-8.html#conditional-types
       // do not break before them, as the expressions are subject to ASI.
       return false;
     }
     if (Left.is(Keywords.kw_as))
       return true;
-    if (Left.is(TT_JsNonNullAssertion))
+    if (Left.is(TT_NonNullAssertion))
       return true;
     if (Left.is(Keywords.kw_declare) &&
         Right.isOneOf(Keywords.kw_module, tok::kw_namespace,
                       Keywords.kw_function, tok::kw_class, tok::kw_enum,
                       Keywords.kw_interface, Keywords.kw_type, Keywords.kw_var,
                       Keywords.kw_let, tok::kw_const))
       // See grammar for 'declare' statements at:
       // https://github.com/Microsoft/TypeScript/blob/master/doc/spec.md#A.10

clang/lib/Format/UnwrappedLineParser.cpp

case tok::identifier: {
           addUnwrappedLine();
           return;
         }
       }
       break;
     }
     case tok::equal:
       // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType
-      // TT_JsFatArrow. The always start an expression or a child block if
+      // TT_FatArrow. The always start an expression or a child block if
       // followed by a curly.
-      if (FormatTok->is(TT_JsFatArrow)) {
+      if (FormatTok->is(TT_FatArrow)) {
         nextToken();
         if (FormatTok->is(tok::l_brace)) {
           // C# may break after => if the next character is a newline.
           if (Style.isCSharp() && Style.BraceWrapping.AfterFunction == true) {
             // calling `addUnwrappedLine()` here causes odd parsing errors.
             FormatTok->MustBreakBefore = true;
           }
           parseChildBlock();
case tok::r_brace:
       return true;
     case tok::l_brace:
       ++Line->Level;
       parseBlock(/*MustBeDeclaration=*/true);
       addUnwrappedLine();
       --Line->Level;
       break;
     case tok::equal:
-      if (FormatTok->is(TT_JsFatArrow)) {
+      if (FormatTok->is(TT_FatArrow)) {
         ++Line->Level;
         do {
           nextToken();
         } while (!eof() && FormatTok->isNot(tok::semi));
         nextToken();
         addUnwrappedLine();
         --Line->Level;
         break;
bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
                                           bool IsEnum,
                                           tok::TokenKind ClosingBraceKind) {
   bool HasError = false;
 
   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
   // replace this by using parseAssigmentExpression() inside.
   do {
     if (Style.isCSharp()) {
-      if (FormatTok->is(TT_JsFatArrow)) {
+      if (FormatTok->is(TT_FatArrow)) {
         nextToken();
         // Fat arrows can be followed by simple expressions or by child blocks
         // in curly braces.
         if (FormatTok->is(tok::l_brace)) {
           parseChildBlock();
           continue;
         }
       }
     }
     if (Style.Language == FormatStyle::LK_JavaScript) {
       if (FormatTok->is(Keywords.kw_function) ||
           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();
         continue;
       }
-      if (FormatTok->is(TT_JsFatArrow)) {
+      if (FormatTok->is(TT_FatArrow)) {
         nextToken();
         // Fat arrows can be followed by simple expressions or by child blocks
         // in curly braces.
         if (FormatTok->is(tok::l_brace)) {
           parseChildBlock();
           continue;
         }
       }