[clang-format] Fix regression merging comments across newlines.

[krasimir]

Summary

This fixes a regression that causes example:

enum A {
  a, // line a

  // line b
  b
};

to be formatted as follows:

enum A {
  a, // line a
     // line b
  b
};

[krasimir]

• Remove newline

[klimek]

lib/Format/BreakableToken.cpp

685

Could we put them as children into the higher level UnwrappedLine instead?

[krasimir]

lib/Format/BreakableToken.cpp

685

Putting them as children looks cleaner; will try that.

[krasimir]

• Add a FIXME for separating line comment sections into child lines

[krasimir]

lib/Format/BreakableToken.cpp

685

For now, I'm adding a FIXME, since this putting the sections into children has some undesired ripple effects in other places and needs more in-depth analysis.

[klimek]

lg minus adding the FIXME to the place where we need the change.

lib/Format/UnwrappedLineParser.cpp

2127–2129

Can you also add a FIXME here (as this is where it'd probably need to happen :)

[krasimir]

• Add a FIXME also in the parser

[krasimir]

lib/Format/UnwrappedLineParser.cpp

2127–2129

Done.

lib/Format/BreakableToken.cpp

for (size_t i = FirstLineIndex, e = Lines.size(); i < e; ++i) {
       size_t EndOfLine = Content[i].find_last_not_of(Blanks);
       if (EndOfLine == StringRef::npos)
         EndOfLine = Content[i].size();
       else
         ++EndOfLine;
       Content[i] = Content[i].substr(0, EndOfLine);
     }
     LineTok = CurrentTok->Next;
+    if (CurrentTok->Next && CurrentTok->Next->NewlinesBefore > 1) {
+      // A line comment section needs to broken by a line comment that is
+      // preceded by at least two newlines. Note that we put this break here
+      // instead of breaking at a previous stage during parsing, since that
+      // would split the contents of the enum into two unwrapped lines in this
+      // example, which is undesirable:
+      // enum A {
+      //   a, // comment about a
+      //
+      //   // comment about b
+      //   b
+      // };
+      //
+      // FIXME: Consider putting separate line comment sections as children to
+      // the unwrapped line instead.
+      break;
+    }
   }
 }
 
 unsigned BreakableLineCommentSection::getLineLengthAfterSplit(
     unsigned LineIndex, unsigned TailOffset,
     StringRef::size_type Length) const {
   unsigned ContentStartColumn =
       (TailOffset == 0 ? ContentColumn[LineIndex]

lib/Format/UnwrappedLineParser.cpp

 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
   bool JustComments = Line->Tokens.empty();
   for (SmallVectorImpl<FormatToken *>::const_iterator
            I = CommentsBeforeNextToken.begin(),
            E = CommentsBeforeNextToken.end();
        I != E; ++I) {
     // Line comments that belong to the same line comment section are put on the
     // same line since later we might want to reflow content between them.
-    // See BreakableToken.
+    // Additional fine-grained breaking of line comment sections is controlled
+    // by the class BreakableLineCommentSection in case it is desirable to keep
+    // several line comment sections in the same unwrapped line.
     if (isOnNewLine(**I) && JustComments && !continuesLineComment(**I, *Line))
       addUnwrappedLine();
     pushToken(*I);
   }
   if (NewlineBeforeNext && JustComments)
     addUnwrappedLine();
   CommentsBeforeNextToken.clear();
 }

unittests/Format/FormatTest.cpp

EXPECT_EQ("#include \"a\"     // comment\n"
                    "#include \"a/b/c\" // comment"));
 
   verifyFormat("enum E {\n"
                "  // comment\n"
                "  VAL_A, // comment\n"
                "  VAL_B\n"
                "};");
 
+  EXPECT_EQ("enum A {\n"
+            "  // line a\n"
+            "  a,\n"
+            "  b, // line b\n"
+            "\n"
+            "  // line c\n"
+            "  c\n"
+            "};",
+            format("enum A {\n"
+                   "  // line a\n"
+                   "  a,\n"
+                   "  b, // line b\n"
+                   "\n"
+                   "  // line c\n"
+                   "  c\n"
+                   "};",
+                   getLLVMStyleWithColumns(20)));
+
   verifyFormat(
       "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa =\n"
       "    bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb; // Trailing comment");
   verifyFormat("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa =\n"
                "    // Comment inside a statement.\n"
                "    bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb;");
   verifyFormat("SomeFunction(a,\n"
                "             // comment\n"