[clang-format] Enable comment reflowing in multiline comments containing pragmas

[krasimir]

Summary

This patch enables comment reflowing of lines not matching the comment pragma regex
in multiline comments containing comment pragma lines. Previously, these comments
were dumped without being reindented to the result.

[krasimir]

• Fix test comment

[mprobst]

LGTM.

[krasimir]

• Make the test example idiomatic

lib/Format/BreakableToken.h

public:
   virtual unsigned
   getLineLengthAfterSplit(unsigned LineIndex, unsigned TailOffset,
                           StringRef::size_type Length) const = 0;
 
   /// \brief Returns a range (offset, length) at which to break the line at
   /// \p LineIndex, if previously broken at \p TailOffset. If possible, do not
   /// violate \p ColumnLimit.
   virtual Split getSplit(unsigned LineIndex, unsigned TailOffset,
-                         unsigned ColumnLimit) const = 0;
+                         unsigned ColumnLimit,
+                         llvm::Regex &CommentPragmasRegex) const = 0;
 
   /// \brief Emits the previously retrieved \p Split via \p Whitespaces.
   virtual void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
                            WhitespaceManager &Whitespaces) = 0;
 
   /// \brief Returns the number of columns required to format the piece of line
   /// at \p LineIndex, from byte offset \p TailOffset after the whitespace range
   /// \p Split has been compressed into a single space.
public:
   ///
   /// \p StartColumn specifies the column in which the token will start
   /// after formatting.
   BreakableStringLiteral(const FormatToken &Tok, unsigned StartColumn,
                          StringRef Prefix, StringRef Postfix,
                          bool InPPDirective, encoding::Encoding Encoding,
                          const FormatStyle &Style);
 
-  Split getSplit(unsigned LineIndex, unsigned TailOffset,
-                 unsigned ColumnLimit) const override;
+  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
+                 llvm::Regex &CommentPragmasRegex) const override;
   void insertBreak(unsigned LineIndex, unsigned TailOffset, Split Split,
                    WhitespaceManager &Whitespaces) override;
   void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,
                           WhitespaceManager &Whitespaces) override {}
 };
 
 class BreakableComment : public BreakableToken {
 protected:
   /// \brief Creates a breakable token for a comment.
   ///
   /// \p StartColumn specifies the column in which the comment will start after
   /// formatting.
   BreakableComment(const FormatToken &Token, unsigned StartColumn,
                    bool InPPDirective, encoding::Encoding Encoding,
                    const FormatStyle &Style);
 
 public:
   unsigned getLineCount() const override;
-  Split getSplit(unsigned LineIndex, unsigned TailOffset,
-                 unsigned ColumnLimit) const override;
+  Split getSplit(unsigned LineIndex, unsigned TailOffset, unsigned ColumnLimit,
+                 llvm::Regex &CommentPragmasRegex) const override;
   void compressWhitespace(unsigned LineIndex, unsigned TailOffset, Split Split,
                           WhitespaceManager &Whitespaces) override;
 
 protected:
   virtual unsigned getContentStartColumn(unsigned LineIndex,
                                          unsigned TailOffset) const = 0;
 
   // Returns a split that divides Text into a left and right parts, such that

lib/Format/BreakableToken.cpp

BreakableStringLiteral::BreakableStringLiteral(
     const FormatToken &Tok, unsigned StartColumn, StringRef Prefix,
     StringRef Postfix, bool InPPDirective, encoding::Encoding Encoding,
     const FormatStyle &Style)
     : BreakableSingleLineToken(Tok, StartColumn, Prefix, Postfix, InPPDirective,
                                Encoding, Style) {}
 
 BreakableToken::Split
 BreakableStringLiteral::getSplit(unsigned LineIndex, unsigned TailOffset,
-                                 unsigned ColumnLimit) const {
+                                 unsigned ColumnLimit,
+                                 llvm::Regex &CommentPragmasRegex) const {
   return getStringSplit(Line.substr(TailOffset),
                         StartColumn + Prefix.size() + Postfix.size(),
                         ColumnLimit, Style.TabWidth, Encoding);
 }
 
 void BreakableStringLiteral::insertBreak(unsigned LineIndex,
                                          unsigned TailOffset, Split Split,
                                          WhitespaceManager &Whitespaces) {
BreakableComment::BreakableComment(const FormatToken &Token,
                                    bool InPPDirective,
                                    encoding::Encoding Encoding,
                                    const FormatStyle &Style)
     : BreakableToken(Token, InPPDirective, Encoding, Style),
       StartColumn(StartColumn) {}
 
 unsigned BreakableComment::getLineCount() const { return Lines.size(); }
 
-BreakableToken::Split BreakableComment::getSplit(unsigned LineIndex,
-                                                 unsigned TailOffset,
-                                                 unsigned ColumnLimit) const {
+BreakableToken::Split
+BreakableComment::getSplit(unsigned LineIndex, unsigned TailOffset,
+                           unsigned ColumnLimit,
+                           llvm::Regex &CommentPragmasRegex) const {
+  // Don't break lines matching the comment pragmas regex.
+  if (CommentPragmasRegex.match(Content[LineIndex]))
+    return Split(StringRef::npos, 0);
   return getCommentSplit(Content[LineIndex].substr(TailOffset),
                          getContentStartColumn(LineIndex, TailOffset),
                          ColumnLimit, Style.TabWidth, Encoding);
 }
 
 void BreakableComment::compressWhitespace(unsigned LineIndex,
                                           unsigned TailOffset, Split Split,
                                           WhitespaceManager &Whitespaces) {

lib/Format/ContinuationIndenter.cpp

if ((Text.endswith(Postfix = "\"") &&
       Token.reset(new BreakableStringLiteral(Current, StartColumn, Prefix,
                                              Postfix, State.Line->InPPDirective,
                                              Encoding, Style));
     } else {
       return 0;
     }
   } else if (Current.is(TT_BlockComment)) {
     if (!Current.isTrailingComment() || !Style.ReflowComments ||
-        CommentPragmasRegex.match(Current.TokenText.substr(2)) ||
         // If a comment token switches formatting, like
         // /* clang-format on */, we don't want to break it further,
         // but we may still want to adjust its indentation.
         switchesFormatting(Current))
       return addMultilineToken(Current, State);
     Token.reset(new BreakableBlockComment(
         Current, StartColumn, Current.OriginalColumn, !Current.Previous,
         State.Line->InPPDirective, Encoding, Style));
for (unsigned LineIndex = 0, EndIndex = Token->getLineCount();
     unsigned TailOffset =
         ReflowInProgress ? (SplitBefore.first + SplitBefore.second) : 0;
     if (!DryRun)
       Token->replaceWhitespaceBefore(LineIndex, RemainingTokenColumns,
                                      RemainingSpace, SplitBefore, Whitespaces);
     RemainingTokenColumns = Token->getLineLengthAfterSplitBefore(
         LineIndex, TailOffset, RemainingTokenColumns, ColumnLimit, SplitBefore);
     while (RemainingTokenColumns > RemainingSpace) {
-      BreakableToken::Split Split =
-          Token->getSplit(LineIndex, TailOffset, ColumnLimit);
+      BreakableToken::Split Split = Token->getSplit(
+          LineIndex, TailOffset, ColumnLimit, CommentPragmasRegex);
       if (Split.first == StringRef::npos) {
         // The last line's penalty is handled in addNextStateToQueue().
         if (LineIndex < EndIndex - 1)
           Penalty += Style.PenaltyExcessCharacter *
                      (RemainingTokenColumns - RemainingSpace);
         break;
       }
       assert(Split.first != 0);

unittests/Format/FormatTestJS.cpp

verifyFormat(
       " * @param This is a\n"
       " * long comment but\n"
       " * no type\n"
       " */",
       "/**\n"
       " * @param This is a long comment but no type\n"
       " */",
       getGoogleJSStyleWithColumns(20));
+  // Don't break @param line, but reindent unrelated lines.
+  verifyFormat("{\n"
+               "  /**\n"
+               "   * long long long\n"
+               "   * long\n"
+               "   * @param {this.is.a.long.path.to.a.Type}\n"
+               "   * long long long\n"
+               "   * long long\n"
+               "   */\n"
+               "  function f(a) {}\n"
+               "}",
+               "{\n"
+               "/**\n"
+               " * long long long long\n"
+               " * @param {this.is.a.long.path.to.a.Type}\n"
+               " * long long long long\n"
+               " * long\n"
+               " */\n"
+               "  function f(a) {}\n"
+               "}",
+               getGoogleJSStyleWithColumns(20));
 }
 
 TEST_F(FormatTestJS, RequoteStringsSingle) {
   verifyFormat("var x = 'foo';", "var x = \"foo\";");
   verifyFormat("var x = 'fo\\'o\\'';", "var x = \"fo'o'\";");
   verifyFormat("var x = 'fo\\'o\\'';", "var x = \"fo\\'o'\";");
   verifyFormat(
       "var x =\n"
verifyFormat("class Foo {\n"
                "}");
 }
 
 TEST_F(FormatTestJS, ImportComments) {
   verifyFormat("import {x} from 'x';  // from some location",
                getGoogleJSStyleWithColumns(25));
   verifyFormat("// taze: x from 'location'", getGoogleJSStyleWithColumns(10));
 }
-
 } // end namespace tooling
 } // end namespace clang