[clang-format] [PR33890] Add support for Microsoft C++/CLI non standard for each looping extension

[MyDeveloperDay]

Summary

https://bugs.llvm.org/show_bug.cgi?id=33890

This revision allow the microsoft for each(.... in ... nonstandard C++ extension which can be used in C++/CLI to be handled as a ForEach macro.

This prevents the breaking between the for and each onto a new line

clang/lib/Format/FormatTokenLexer.h

private:
   bool tryMergeLessLess();
   bool tryMergeNSStringLiteral();
   bool tryMergeJSPrivateIdentifier();
   bool tryMergeCSharpStringLiteral();
   bool tryMergeCSharpKeywordVariables();
   bool tryMergeCSharpDoubleQuestion();
   bool tryMergeCSharpNullConditional();
   bool tryTransformCSharpForEach();
+  bool tryMergeForEach();
 
   bool tryMergeTokens(ArrayRef<tok::TokenKind> Kinds, TokenType NewType);
 
   // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
   bool precedesOperand(FormatToken *Tok);
 
   bool canPrecedeRegexLiteral(FormatToken *Prev);
 

clang/lib/Format/FormatTokenLexer.cpp

 
 void FormatTokenLexer::tryMergePreviousTokens() {
   if (tryMerge_TMacro())
     return;
   if (tryMergeConflictMarkers())
     return;
   if (tryMergeLessLess())
     return;
+  if (tryMergeForEach())
+    return;
 
   if (Style.isCSharp()) {
     if (tryMergeCSharpKeywordVariables())
       return;
     if (tryMergeCSharpStringLiteral())
       return;
     if (tryMergeCSharpDoubleQuestion())
       return;
bool FormatTokenLexer::tryTransformCSharpForEach() {
   if (Identifier->TokenText != "foreach")
     return false;
 
   Identifier->setType(TT_ForEachMacro);
   Identifier->Tok.setKind(tok::kw_for);
   return true;
 }
 
+bool FormatTokenLexer::tryMergeForEach() {
+  if (Tokens.size() < 2)
+    return false;
+  auto &For = *(Tokens.end() - 2);
+  auto &Each = *(Tokens.end() - 1);
+  if (!For->is(tok::kw_for))
+    return false;
+  if (!Each->is(tok::identifier))
+    return false;
+  if (Each->TokenText != "each")
+    return false;
+
+  For->setType(TT_ForEachMacro);
+  For->Tok.setKind(tok::kw_for);
+
+  For->TokenText = StringRef(For->TokenText.begin(),
+                             Each->TokenText.end() - For->TokenText.begin());
+  For->ColumnWidth += Each->ColumnWidth;
+  Tokens.erase(Tokens.end() - 1);
+  return true;
+}
+
 bool FormatTokenLexer::tryMergeLessLess() {
   // Merge X,less,less,Y into X,lessless,Y unless X or Y is less.
   if (Tokens.size() < 3)
     return false;
 
   bool FourthTokenIsLess = false;
   if (Tokens.size() > 3)
     FourthTokenIsLess = (Tokens.end() - 4)[0]->is(tok::less);

clang/unittests/Format/FormatTest.cpp

verifyFormat("#define foreach(x, y)\n"
                "#define BOOST_FOREACH(x, y)\n"
                "#define UNKNOWN_FOREACH(x, y)\n");
 
   // Not as function-like macros.
   verifyFormat("#define foreach (x, y)\n"
                "#define Q_FOREACH (x, y)\n"
                "#define BOOST_FOREACH (x, y)\n"
                "#define UNKNOWN_FOREACH (x, y)\n");
+
+  // handle microsoft non standard extension
+  verifyFormat("for each (char c in x->MyStringProperty)");
 }
 
 TEST_F(FormatTest, FormatsWhileLoop) {
   verifyFormat("while (true) {\n}");
   verifyFormat("while (true)\n"
                "  f();");
   verifyFormat("while () {\n}");
   verifyFormat("while () {\n"