[clang-format] Handle C# generic type constraints

[jbcoe]

Summary

Treat each C# generic type constraint, where T: ..., as a line.

Add C# keyword: where

Add Token Types: CSharpGenericTypeConstraint, CSharpGenericTypeConstraintColon, CSharpGenericTypeConstraintComma.

This patch does not wrap generic type constraints well, that will be addressed in a follow up patch.

[krasimir]

clang/lib/Format/UnwrappedLineParser.cpp

326

If you remove the reference from parseRecored (see comment there), consider removing the ConsumeBrace argument -- all other references set it to false.
We can add this later when we need it.

2310

No tests fail if I remove this if.
Consider adding a test case for this or removing.
As this is the first in a series of patches, maybe we can add this in a later patch.

clang/unittests/Format/FormatTestCSharp.cpp

631

consider moving this test case to the new CSharpGenericTypeConstraints, re-formatted (as I understand it, this patch forces a newline before the while).

[jbcoe]

Address review comments to move deleted test and remove redundant code.

Redundant call to parseCSharpGenericTypeConstraint was used for free functions which are not legal in C#.

[krasimir]

Thank you!

clang/lib/Format/FormatToken.h

#define LIST_TOKEN_TYPES                                                       \
   TYPE(TypenameMacro)                                                          \
   TYPE(UnaryOperator)                                                          \
   TYPE(CSharpStringLiteral)                                                    \
   TYPE(CSharpNamedArgumentColon)                                               \
   TYPE(CSharpNullable)                                                         \
   TYPE(CSharpNullCoalescing)                                                   \
   TYPE(CSharpNullConditional)                                                  \
   TYPE(CSharpNullConditionalLSquare)                                           \
+  TYPE(CSharpGenericTypeConstraint)                                            \
+  TYPE(CSharpGenericTypeConstraintColon)                                       \
+  TYPE(CSharpGenericTypeConstraintComma)                                       \
   TYPE(Unknown)
 
 enum TokenType {
 #define TYPE(X) TT_##X,
   LIST_TOKEN_TYPES
 #undef TYPE
       NUM_TOKEN_TYPES
 };
AdditionalKeywords(IdentifierTable &IdentTable) {
     kw_sbyte = &IdentTable.get("sbyte");
     kw_sealed = &IdentTable.get("sealed");
     kw_uint = &IdentTable.get("uint");
     kw_ulong = &IdentTable.get("ulong");
     kw_unchecked = &IdentTable.get("unchecked");
     kw_unsafe = &IdentTable.get("unsafe");
     kw_ushort = &IdentTable.get("ushort");
     kw_when = &IdentTable.get("when");
+    kw_where = &IdentTable.get("where");
 
     // Keep this at the end of the constructor to make sure everything here
     // is
     // already initialized.
     JsExtraKeywords = std::unordered_set<IdentifierInfo *>(
         {kw_as, kw_async, kw_await, kw_declare, kw_finally, kw_from,
          kw_function, kw_get, kw_import, kw_is, kw_let, kw_module, kw_readonly,
          kw_set, kw_type, kw_typeof, kw_var, kw_yield,
          // Keywords from the Java section.
          kw_abstract, kw_extends, kw_implements, kw_instanceof, kw_interface});
 
     CSharpExtraKeywords = std::unordered_set<IdentifierInfo *>(
         {kw_base, kw_byte, kw_checked, kw_decimal, kw_delegate, kw_event,
          kw_fixed, kw_foreach, kw_implicit, kw_in, kw_interface, kw_internal,
          kw_is, kw_lock, kw_null, kw_object, kw_out, kw_override, kw_params,
          kw_readonly, kw_ref, kw_string, kw_stackalloc, kw_sbyte, kw_sealed,
          kw_uint, kw_ulong, kw_unchecked, kw_unsafe, kw_ushort, kw_when,
+         kw_where,
          // Keywords from the JavaScript section.
          kw_as, kw_async, kw_await, kw_declare, kw_finally, kw_from,
          kw_function, kw_get, kw_import, kw_is, kw_let, kw_module, kw_readonly,
          kw_set, kw_type, kw_typeof, kw_var, kw_yield,
          // Keywords from the Java section.
          kw_abstract, kw_extends, kw_implements, kw_instanceof, kw_interface});
   }
 
struct AdditionalKeywords {
   IdentifierInfo *kw_sbyte;
   IdentifierInfo *kw_sealed;
   IdentifierInfo *kw_uint;
   IdentifierInfo *kw_ulong;
   IdentifierInfo *kw_unchecked;
   IdentifierInfo *kw_unsafe;
   IdentifierInfo *kw_ushort;
   IdentifierInfo *kw_when;
+  IdentifierInfo *kw_where;
 
   /// Returns \c true if \p Tok is a true JavaScript identifier, returns
   /// \c false if it is a keyword or a pseudo keyword.
   bool IsJavaScriptIdentifier(const FormatToken &Tok) const {
     return Tok.is(tok::identifier) &&
            JsExtraKeywords.find(Tok.Tok.getIdentifierInfo()) ==
                JsExtraKeywords.end();
   }

clang/lib/Format/TokenAnnotator.cpp

case tok::comma:
       if (Contexts.back().IsForEachMacro)
         Contexts.back().IsExpression = true;
       break;
     case tok::identifier:
       if (Tok->isOneOf(Keywords.kw___has_include,
                        Keywords.kw___has_include_next)) {
         parseHasInclude();
       }
+      if (Tok->is(Keywords.kw_where) && Tok->Next &&
+          Tok->Next->isNot(tok::l_paren)) {
+        Tok->Type = TT_CSharpGenericTypeConstraint;
+        parseCSharpGenericTypeConstraint();
+      }
       break;
     default:
       break;
     }
     return true;
   }
 
+  void parseCSharpGenericTypeConstraint() {
+    while (CurrentToken) {
+      if (CurrentToken->is(tok::less)) {
+        // parseAngle is too greedy and will consume the whole line.
+        CurrentToken->Type = TT_TemplateOpener;
+        next();
+      } else if (CurrentToken->is(tok::greater)) {
+        CurrentToken->Type = TT_TemplateCloser;
+        next();
+      } else if (CurrentToken->is(tok::comma)) {
+        CurrentToken->Type = TT_CSharpGenericTypeConstraintComma;
+        next();
+      } else if (CurrentToken->is(Keywords.kw_where)) {
+        CurrentToken->Type = TT_CSharpGenericTypeConstraint;
+        next();
+      } else if (CurrentToken->is(tok::colon)) {
+        CurrentToken->Type = TT_CSharpGenericTypeConstraintColon;
+        next();
+      } else {
+        next();
+      }
+    }
+  }
+
   void parseIncludeDirective() {
     if (CurrentToken && CurrentToken->is(tok::less)) {
       next();
       while (CurrentToken) {
         // Mark tokens up to the trailing line comments as implicit string
         // literals.
         if (CurrentToken->isNot(tok::comment) &&
             !CurrentToken->TokenText.startswith("//"))
bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   const FormatToken &Left = *Right.Previous;
   if (Right.NewlinesBefore > 1 && Style.MaxEmptyLinesToKeep > 0)
     return true;
 
   if (Style.isCSharp()) {
     if (Right.is(TT_CSharpNamedArgumentColon) ||
         Left.is(TT_CSharpNamedArgumentColon))
       return false;
+    if (Right.is(TT_CSharpGenericTypeConstraint))
+      return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     // FIXME: This might apply to other languages and token kinds.
     if (Right.is(tok::string_literal) && Left.is(tok::plus) && Left.Previous &&
         Left.Previous->is(tok::string_literal))
       return true;
     if (Left.is(TT_DictLiteral) && Left.is(tok::l_brace) && Line.Level == 0 &&
         Left.Previous && Left.Previous->is(tok::equal) &&
         Line.First->isOneOf(tok::identifier, Keywords.kw_import, tok::kw_export,

clang/lib/Format/UnwrappedLineFormatter.cpp

void nextLine(const AnnotatedLine &Line) {
     if (Line.InPPDirective) {
       Indent = Line.Level * Style.IndentWidth + AdditionalIndent;
     } else {
       IndentForLevel.resize(Line.Level + 1);
       Indent = getIndent(IndentForLevel, Line.Level);
     }
     if (static_cast<int>(Indent) + Offset >= 0)
       Indent += Offset;
+    if (Line.First->is(TT_CSharpGenericTypeConstraint))
+      Indent = Line.Level * Style.IndentWidth + Style.ContinuationIndentWidth;
   }
 
   /// Update the indent state given that \p Line indent should be
   /// skipped.
   void skipLine(const AnnotatedLine &Line) {
     while (IndentForLevel.size() <= Line.Level)
       IndentForLevel.push_back(Indent);
   }

clang/lib/Format/UnwrappedLineParser.h

private:
   void parseObjCMethod();
   void parseObjCProtocolList();
   void parseObjCUntilAtEnd();
   void parseObjCInterfaceOrImplementation();
   bool parseObjCProtocol();
   void parseJavaScriptEs6ImportExport();
   void parseStatementMacro();
   void parseCSharpAttribute();
+  // Parse a C# generic type constraint: `where T : IComparable<T>`.
+  // See:
+  // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/where-generic-type-constraint
+  void parseCSharpGenericTypeConstraint();
   bool tryToParseLambda();
   bool tryToParseLambdaIntroducer();
   void tryToParseJSFunction();
   void addUnwrappedLine();
   bool eof() const;
   // LevelDifference is the difference of levels after and before the current
   // token. For example:
   // - if the token is '{' and opens a block, LevelDifference is 1.

clang/lib/Format/UnwrappedLineParser.cpp

void UnwrappedLineParser::parseFile() {
   // Here we add this newline before end-of-file comments.
   if (Style.Language == FormatStyle::LK_TextProto &&
       !CommentsBeforeNextToken.empty())
     addUnwrappedLine();
   flushComments(true);
   addUnwrappedLine();
 }
 
+void UnwrappedLineParser::parseCSharpGenericTypeConstraint() {
+  do {
+    switch (FormatTok->Tok.getKind()) {
+    case tok::l_brace:
+      return;
+    default:
+      if (FormatTok->is(Keywords.kw_where)) {
+        addUnwrappedLine();
+        nextToken();
+        parseCSharpGenericTypeConstraint();
+        break;
+      }
+      nextToken();
+      break;
+    }
+  } while (!eof());
+}
+
 void UnwrappedLineParser::parseCSharpAttribute() {
   int UnpairedSquareBrackets = 1;
   do {
     switch (FormatTok->Tok.getKind()) {
     case tok::r_square:
       nextToken();
       --UnpairedSquareBrackets;
       if (UnpairedSquareBrackets == 0) {
case tok::kw_try:
         break;
       }
       // We arrive here when parsing function-try blocks.
       if (Style.BraceWrapping.AfterFunction)
         addUnwrappedLine();
       parseTryCatch();
       return;
     case tok::identifier: {
+      if (Style.isCSharp() && FormatTok->is(Keywords.kw_where) &&
+          Line->MustBeDeclaration) {
+        addUnwrappedLine();
+        parseCSharpGenericTypeConstraint();
+        break;
+      }
       if (FormatTok->is(TT_MacroBlockEnd)) {
         addUnwrappedLine();
         return;
       }
 
       // Function declarations (as opposed to function expressions) are parsed
       // on their own unwrapped line by continuing this loop. Function
       // expressions (functions that are not on their own line) must not create
if (Style.Language == FormatStyle::LK_JavaScript &&
       // extends/implements positions:
       //     class Foo implements {bar: number} { }
       nextToken();
       if (FormatTok->is(tok::l_brace)) {
         tryToParseBracedList();
         continue;
       }
     }
     bool IsNonMacroIdentifier =
         FormatTok->is(tok::identifier) &&
         FormatTok->TokenText != FormatTok->TokenText.upper();
     nextToken();
     // We can have macros or attributes in between 'class' and the class name.
     if (!IsNonMacroIdentifier && FormatTok->Tok.is(tok::l_paren))
       parseParens();
   }
 

clang/unittests/Format/FormatTestCSharp.cpp

TEST_F(FormatTestCSharp, CSharpSpaces) {
   verifyFormat(R"(new Dictionary<string, string> { ["Key"] = "Value" };)",
                Style);
   verifyFormat(R"(Apply(x => x.Name, x => () => x.ID);)", Style);
   verifyFormat(R"(bool[] xs = { true, true };)", Style);
   verifyFormat(R"(taskContext.Factory.Run(async () => doThing(args);)", Style);
   verifyFormat(R"(catch (TestException) when (innerFinallyExecuted))", Style);
   verifyFormat(R"(private float[,] Values;)", Style);
   verifyFormat(R"(Result this[Index x] => Foo(x);)", Style);
-  verifyFormat(R"(class ItemFactory<T> where T : new() {})", Style);
 
   Style.SpacesInSquareBrackets = true;
   verifyFormat(R"(private float[ , ] Values;)", Style);
   verifyFormat(R"(string dirPath = args?[ 0 ];)", Style);
 }
 
 TEST_F(FormatTestCSharp, CSharpNullableTypes) {
   FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
if (someThings[index].Contains(myThing)) {
                Style);
 
   verifyFormat(R"(//
 if (someThings[i][j][k].Contains(myThing)) {
 })",
                Style);
 }
 
+TEST_F(FormatTestCSharp, CSharpGenericTypeConstraints) {
+  FormatStyle Style = getGoogleStyle(FormatStyle::LK_CSharp);
+
+  verifyFormat(R"(//
+class ItemFactory<T>
+    where T : new() {})", Style);
+
+  verifyFormat(R"(//
+class Dictionary<TKey, TVal>
+    where TKey : IComparable<TKey>
+    where TVal : IMyInterface {
+  public void MyMethod<T>(T t)
+      where T : IMyInterface { doThing(); }
+})",
+               Style);
+}
+
 } // namespace format
 } // end namespace clang