[clang-format] Add ability to trace tokens.

[klimek]

Remove superfluous include.

[sammccall]

clang/lib/Format/ContinuationIndenter.cpp

289

annotating all exit paths from this function and mustBreak seems much more verbose and fragile than wrapping the function (making this version private) and adding the annotations in the wrapper.

clang/lib/Format/FormatDebug.h

23

it looks like you're not providing a dummy definition in NDEBUG mode - does this still build in that config?

29

Comment with an example?
(In particular, unclear what "Debug" means)

33

why empty else?

clang/lib/Format/TokenAnnotator.cpp

3258

move this out of the if() and remove the one on the other branch?

3258

this meaning of << in (...) << "MustBreakBefore is confusing.

consider << (...) << "MustBreakBefore" or dbgs() << (...) << "MustBreakBefore" or a twine or formatv or really anything that isn't a shift :-)

[klimek]

Answering the fundamental design question first.

clang/lib/Format/ContinuationIndenter.cpp

289

How do we get the exact condition that triggered? The main trick here is the FILE:LINE in the debug output.

clang/lib/Format/FormatDebug.h

23

All use is within LLVM_DEBUG().

33

The idea was dangling else protection, but I guess we have dangling else warnings, so this is not necessary?

clang/lib/Format/TokenAnnotator.cpp

3258

The idea was to see which branch of the if is taken here.

[sammccall]

Oops, sorry for the noise.

I'd forgotten that we're really interested in *why* we reach the conclusion, not the conclusion itself - and that file/line are our proxy for "why" as writing/maintaining descriptions is a burden.

clang/lib/Format/ContinuationIndenter.cpp

289

Doh. There *is* an a way in recent clang/gcc/msvc, though I suspect you don't want to go down this path...

template <typename T>
struct LogReturn {
  T Value;
  #ifdef NDEBUG
  LogReturn(T Value) : Value(Value) {}
  #else
  const char *File;
  int Line;
  // default args are evaluated at callsite
  LogReturn(T Value, const char *File=__builtin_FILE(), int Line=__builtin_LINE()) : Value(Value), File(File), Line(Line) {
    dbgs() << ...;
  }
  #endif
}

LogReturn<bool> priv() { return true; }
bool pub() { return priv().Value; }

Dunno how to get the token without resorting to TLS though. More realistically you could define trace(bool, Token&, const char*File=..., etc) and return trace(true, Tok);

clang/lib/Format/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS support)
 
 add_clang_library(clangFormat
   AffectedRangeManager.cpp
   BreakableToken.cpp
   ContinuationIndenter.cpp
   DefinitionBlockSeparator.cpp
   Format.cpp
+  FormatDebug.cpp
   FormatToken.cpp
   FormatTokenLexer.cpp
   IntegerLiteralSeparatorFixer.cpp
   MacroCallReconstructor.cpp
   MacroExpander.cpp
   NamespaceEndCommentsFixer.cpp
   QualifierAlignmentFixer.cpp
   SortJavaScriptImports.cpp

clang/lib/Format/ContinuationIndenter.cpp

 //===--- ContinuationIndenter.cpp - Format C++ code -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
 /// This file implements the continuation indenter.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "ContinuationIndenter.h"
 #include "BreakableToken.h"
+#include "FormatDebug.h"
 #include "FormatInternal.h"
 #include "FormatToken.h"
 #include "WhitespaceManager.h"
 #include "clang/Basic/OperatorPrecedence.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Format/Format.h"
 #include "llvm/ADT/StringSet.h"

 
 bool ContinuationIndenter::canBreak(const LineState &State) {
   const FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *Current.Previous;
   const auto &CurrentState = State.Stack.back();
   assert(&Previous == Current.Previous);
   if (!Current.CanBreakBefore && !(CurrentState.BreakBeforeClosingBrace &&
                                    Current.closesBlockOrBlockTypeList(Style))) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
   }
   // The opening "{" of a braced list has to be on the same line as the first
   // element if it is nested in another braced init list or function call.
   if (!Current.MustBreakBefore && Previous.is(tok::l_brace) &&
       Previous.isNot(TT_DictLiteral) && Previous.is(BK_BracedInit) &&
       Previous.Previous &&
       Previous.Previous->isOneOf(tok::l_brace, tok::l_paren, tok::comma)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
   }
   // This prevents breaks like:
   //   ...
   //   SomeParameter, OtherParameter).DoSomething(
   //   ...
   // As they hide "DoSomething" and are generally bad for readability.
   if (Previous.opensScope() && Previous.isNot(tok::l_brace) &&
       State.LowestLevelOnLine < State.StartOfLineLevel &&
       State.LowestLevelOnLine < Current.NestingLevel) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
   }
-  if (Current.isMemberAccess() && CurrentState.ContainsUnwrappedBuilder)
+  if (Current.isMemberAccess() && CurrentState.ContainsUnwrappedBuilder) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
+  }
 
   // Don't create a 'hanging' indent if there are multiple blocks in a single
   // statement.
   if (Previous.is(tok::l_brace) && State.Stack.size() > 1 &&
       State.Stack[State.Stack.size() - 2].NestedBlockInlined &&
       State.Stack[State.Stack.size() - 2].HasMultipleNestedBlocks) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
   }
 
   // Don't break after very short return types (e.g. "void") as that is often
   // unexpected.
   if (Current.is(TT_FunctionDeclarationName) && State.Column < 6) {
-    if (Style.AlwaysBreakAfterReturnType == FormatStyle::RTBS_None)
+    if (Style.AlwaysBreakAfterReturnType == FormatStyle::RTBS_None) {
+      DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
       return false;
     }
+  }
 
   // If binary operators are moved to the next line (including commas for some
   // styles of constructor initializers), that's always ok.
   if (!Current.isOneOf(TT_BinaryOperator, tok::comma) &&
       CurrentState.NoLineBreakInOperand) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
   }
 
-  if (Previous.is(tok::l_square) && Previous.is(TT_ObjCMethodExpr))
+  if (Previous.is(tok::l_square) && Previous.is(TT_ObjCMethodExpr)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "!canBreak");
     return false;
+  }
 
   if (Current.is(TT_ConditionalExpr) && Previous.is(tok::r_paren) &&
       Previous.MatchingParen && Previous.MatchingParen->Previous &&
       Previous.MatchingParen->Previous->MatchingParen &&
       Previous.MatchingParen->Previous->MatchingParen->is(TT_LambdaLBrace)) {
     // We have a lambda within a conditional expression, allow breaking here.
     assert(Previous.MatchingParen->Previous->is(tok::r_brace));
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "canBreak");
     return true;
   }
 
+  DEBUG_FORMAT_TRACE_TOKEN(Current, (CurrentState.NoLineBreak ? "!" : "")
+                                        << "canBreak");
   return !CurrentState.NoLineBreak;
 }
 
 bool ContinuationIndenter::mustBreak(const LineState &State) {
   const FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *Current.Previous;
   const auto &CurrentState = State.Stack.back();
   if (Style.BraceWrapping.BeforeLambdaBody && Current.CanBreakBefore &&
       Current.is(TT_LambdaLBrace) && Previous.isNot(TT_LineComment)) {
     auto LambdaBodyLength = getLengthToMatchingParen(Current, State.Stack);
+    DEBUG_FORMAT_TRACE_TOKEN(
+        Current, (LambdaBodyLength <= getColumnLimit(State) ? "!" : "")
+                     << "mustBreak");
     return LambdaBodyLength > getColumnLimit(State);
   }
   if (Current.MustBreakBefore ||
       (Current.is(TT_InlineASMColon) &&
        (Style.BreakBeforeInlineASMColon == FormatStyle::BBIAS_Always ||
         Style.BreakBeforeInlineASMColon == FormatStyle::BBIAS_OnlyMultiline))) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
   if (CurrentState.BreakBeforeClosingBrace &&
       Current.closesBlockOrBlockTypeList(Style)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
-  if (CurrentState.BreakBeforeClosingParen && Current.is(tok::r_paren))
+  if (CurrentState.BreakBeforeClosingParen && Current.is(tok::r_paren)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
+  }
   if (Style.Language == FormatStyle::LK_ObjC &&
       Style.ObjCBreakBeforeNestedBlockParam &&
       Current.ObjCSelectorNameParts > 1 &&
       Current.startsSequence(TT_SelectorName, tok::colon, tok::caret)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
   // Avoid producing inconsistent states by requiring breaks where they are not
   // permitted for C# generic type constraints.
   if (CurrentState.IsCSharpGenericTypeConstraint &&
       Previous.isNot(TT_CSharpGenericTypeConstraintComma)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return false;
   }
   if ((startsNextParameter(Current, Style) || Previous.is(tok::semi) ||
        (Previous.is(TT_TemplateCloser) && Current.is(TT_StartOfName) &&
         Style.isCpp() &&
         // FIXME: This is a temporary workaround for the case where clang-format
         // sets BreakBeforeParameter to avoid bin packing and this creates a
         // completely unnecessary line break after a template type that isn't
         // line-wrapped.
         (Previous.NestingLevel == 1 || Style.BinPackParameters)) ||
        (Style.BreakBeforeTernaryOperators && Current.is(TT_ConditionalExpr) &&
         Previous.isNot(tok::question)) ||
        (!Style.BreakBeforeTernaryOperators &&
         Previous.is(TT_ConditionalExpr))) &&
       CurrentState.BreakBeforeParameter && !Current.isTrailingComment() &&
       !Current.isOneOf(tok::r_paren, tok::r_brace)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
   if (CurrentState.IsChainedConditional &&
       ((Style.BreakBeforeTernaryOperators && Current.is(TT_ConditionalExpr) &&
         Current.is(tok::colon)) ||
        (!Style.BreakBeforeTernaryOperators && Previous.is(TT_ConditionalExpr) &&
         Previous.is(tok::colon)))) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
   if (((Previous.is(TT_DictLiteral) && Previous.is(tok::l_brace)) ||
        (Previous.is(TT_ArrayInitializerLSquare) &&
         Previous.ParameterCount > 1) ||
        opensProtoMessageField(Previous, Style)) &&
       Style.ColumnLimit > 0 &&
       getLengthToMatchingParen(Previous, State.Stack) + State.Column - 1 >
           getColumnLimit(State)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   const FormatToken &BreakConstructorInitializersToken =
       Style.BreakConstructorInitializers == FormatStyle::BCIS_AfterColon
           ? Previous
           : Current;
   if (BreakConstructorInitializersToken.is(TT_CtorInitializerColon) &&
       (State.Column + State.Line->Last->TotalLength - Previous.TotalLength >
            getColumnLimit(State) ||
        CurrentState.BreakBeforeParameter) &&
       (!Current.isTrailingComment() || Current.NewlinesBefore > 0) &&
       (Style.AllowShortFunctionsOnASingleLine != FormatStyle::SFS_All ||
        Style.BreakConstructorInitializers != FormatStyle::BCIS_BeforeColon ||
        Style.ColumnLimit != 0)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   if (Current.is(TT_ObjCMethodExpr) && !Previous.is(TT_SelectorName) &&
       State.Line->startsWith(TT_ObjCMethodSpecifier)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
   if (Current.is(TT_SelectorName) && !Previous.is(tok::at) &&
       CurrentState.ObjCSelectorNameFound && CurrentState.BreakBeforeParameter &&
       (Style.ObjCBreakBeforeNestedBlockParam ||
        !Current.startsSequence(TT_SelectorName, tok::colon, tok::caret))) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   unsigned NewLineColumn = getNewLineColumn(State);
   if (Current.isMemberAccess() && Style.ColumnLimit != 0 &&
       State.Column + getLengthToNextOperator(Current) > Style.ColumnLimit &&
       (State.Column > NewLineColumn ||
        Current.NestingLevel < State.StartOfLineLevel)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   if (startsSegmentOfBuilderTypeCall(Current) &&
       (CurrentState.CallContinuation != 0 ||
        CurrentState.BreakBeforeParameter) &&
       // JavaScript is treated different here as there is a frequent pattern:
       //   SomeFunction(function() {
       //     ...
       //   }.bind(...));
       // FIXME: We should find a more generic solution to this problem.
       !(State.Column <= NewLineColumn && Style.isJavaScript()) &&
       !(Previous.closesScopeAfterBlock() && State.Column <= NewLineColumn)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   // If the template declaration spans multiple lines, force wrap before the
   // function/class declaration.
   if (Previous.ClosesTemplateDeclaration && CurrentState.BreakBeforeParameter &&
       Current.CanBreakBefore) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
-  if (!State.Line->First->is(tok::kw_enum) && State.Column <= NewLineColumn)
+  if (!State.Line->First->is(tok::kw_enum) && State.Column <= NewLineColumn) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return false;
+  }
 
   if (Style.AlwaysBreakBeforeMultilineStrings &&
       (NewLineColumn == State.FirstIndent + Style.ContinuationIndentWidth ||
        Previous.is(tok::comma) || Current.NestingLevel < 2) &&
       !Previous.isOneOf(tok::kw_return, tok::lessless, tok::at,
                         Keywords.kw_dollar) &&
       !Previous.isOneOf(TT_InlineASMColon, TT_ConditionalExpr) &&
       nextIsMultilineString(State)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   // Using CanBreakBefore here and below takes care of the decision whether the
   // current style uses wrapping before or after operators for the given
   // operator.
   if (Previous.is(TT_BinaryOperator) && Current.CanBreakBefore) {
     const auto PreviousPrecedence = Previous.getPrecedence();
     if (PreviousPrecedence != prec::Assignment &&
         CurrentState.BreakBeforeParameter && !Current.isTrailingComment()) {
       const bool LHSIsBinaryExpr =
           Previous.Previous && Previous.Previous->EndsBinaryExpression;
-      if (LHSIsBinaryExpr)
+      if (LHSIsBinaryExpr) {
+        DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
         return true;
+      }
       // If we need to break somewhere inside the LHS of a binary expression, we
       // should also break after the operator. Otherwise, the formatting would
       // hide the operator precedence, e.g. in:
       //   if (aaaaaaaaaaaaaa ==
       //           bbbbbbbbbbbbbb && c) {..
       // For comparisons, we only apply this rule, if the LHS is a binary
       // expression itself as otherwise, the line breaks seem superfluous.
       // We need special cases for ">>" which we have split into two ">" while
       // lexing in order to make template parsing easier.
       const bool IsComparison =
           (PreviousPrecedence == prec::Relational ||
            PreviousPrecedence == prec::Equality ||
            PreviousPrecedence == prec::Spaceship) &&
           Previous.Previous &&
           Previous.Previous->isNot(TT_BinaryOperator); // For >>.
-      if (!IsComparison)
+      if (!IsComparison) {
+        DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
         return true;
       }
+    }
   } else if (Current.is(TT_BinaryOperator) && Current.CanBreakBefore &&
              CurrentState.BreakBeforeParameter) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   // Same as above, but for the first "<<" operator.
   if (Current.is(tok::lessless) && Current.isNot(TT_OverloadedOperator) &&
       CurrentState.BreakBeforeParameter && CurrentState.FirstLessLess == 0) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   if (Current.NestingLevel == 0 && !Current.isTrailingComment()) {
     // Always break after "template <...>"(*) and leading annotations. This is
     // only for cases where the entire line does not fit on a single line as a
     // different LineFormatter would be used otherwise.
     // *: Except when another option interferes with that, like concepts.
     if (Previous.ClosesTemplateDeclaration) {
       if (Current.is(tok::kw_concept)) {
         switch (Style.BreakBeforeConceptDeclarations) {
         case FormatStyle::BBCDS_Allowed:
           break;
         case FormatStyle::BBCDS_Always:
+          DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
           return true;
         case FormatStyle::BBCDS_Never:
+          DEBUG_FORMAT_TRACE_TOKEN(Current, "!mustBreak");
           return false;
         }
       }
       if (Current.is(TT_RequiresClause)) {
         switch (Style.RequiresClausePosition) {
         case FormatStyle::RCPS_SingleLine:
         case FormatStyle::RCPS_WithPreceding:
+          DEBUG_FORMAT_TRACE_TOKEN(Current, "!mustBreak");
           return false;
         default:
+          DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
           return true;
         }
       }
       return Style.AlwaysBreakTemplateDeclarations != FormatStyle::BTDS_No;
     }
     if (Previous.is(TT_FunctionAnnotationRParen) &&
         State.Line->Type != LT_PreprocessorDirective) {
+      DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
       return true;
     }
     if (Previous.is(TT_LeadingJavaAnnotation) && Current.isNot(tok::l_paren) &&
         Current.isNot(TT_LeadingJavaAnnotation)) {
+      DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
       return true;
     }
   }
 
   if (Style.isJavaScript() && Previous.is(tok::r_paren) &&
       Previous.is(TT_JavaAnnotation)) {
     // Break after the closing parenthesis of TypeScript decorators before
     // functions, getters and setters.
     static const llvm::StringSet<> BreakBeforeDecoratedTokens = {"get", "set",
                                                                  "function"};
-    if (BreakBeforeDecoratedTokens.contains(Current.TokenText))
+    if (BreakBeforeDecoratedTokens.contains(Current.TokenText)) {
+      DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
       return true;
     }
+  }
 
   // If the return type spans multiple lines, wrap before the function name.
   if (((Current.is(TT_FunctionDeclarationName) &&
         !State.Line->ReturnTypeWrapped &&
         // Don't break before a C# function when no break after return type.
         (!Style.isCSharp() ||
          Style.AlwaysBreakAfterReturnType != FormatStyle::RTBS_None) &&
         // Don't always break between a JavaScript `function` and the function
         // name.
         !Style.isJavaScript()) ||
        (Current.is(tok::kw_operator) && !Previous.is(tok::coloncolon))) &&
       !Previous.is(tok::kw_template) && CurrentState.BreakBeforeParameter) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   // The following could be precomputed as they do not depend on the state.
   // However, as they should take effect only if the UnwrappedLine does not fit
   // into the ColumnLimit, they are checked here in the ContinuationIndenter.
   if (Style.ColumnLimit != 0 && Previous.is(BK_Block) &&
       Previous.is(tok::l_brace) &&
       !Current.isOneOf(tok::r_brace, tok::comment)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
   if (Current.is(tok::lessless) &&
       ((Previous.is(tok::identifier) && Previous.TokenText == "endl") ||
        (Previous.Tok.isLiteral() && (Previous.TokenText.endswith("\\n\"") ||
                                      Previous.TokenText == "\'\\n\'")))) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
   }
 
-  if (Previous.is(TT_BlockComment) && Previous.IsMultiline)
+  if (Previous.is(TT_BlockComment) && Previous.IsMultiline) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
+  }
 
-  if (State.NoContinuation)
+  if (State.NoContinuation) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "mustBreak");
     return true;
+  }
 
+  DEBUG_FORMAT_TRACE_TOKEN(Current, "!mustBreak");
   return false;
 }
 
 unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
                                                bool DryRun,
                                                unsigned ExtraSpaces) {
   const FormatToken &Current = *State.NextToken;
   assert(State.NextToken->Previous);
if (!DryRun) {
     Whitespaces.replaceWhitespace(Current, /*Newlines=*/0, Spaces,
                                   State.Column + Spaces + PPColumnCorrection);
   }
 
   // If "BreakBeforeInheritanceComma" mode, don't break within the inheritance
   // declaration unless there is multiple inheritance.
   if (Style.BreakInheritanceList == FormatStyle::BILS_BeforeComma &&
       Current.is(TT_InheritanceColon)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
   }
   if (Style.BreakInheritanceList == FormatStyle::BILS_AfterColon &&
       Previous.is(TT_InheritanceColon)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
   }
 
   if (Current.is(TT_SelectorName) && !CurrentState.ObjCSelectorNameFound) {
     unsigned MinIndent = std::max(
         State.FirstIndent + Style.ContinuationIndentWidth, CurrentState.Indent);
     unsigned FirstColonPos = State.Column + Spaces + Current.ColumnWidth;
     if (Current.LongestObjCSelectorName == 0)
if ((Style.AlignAfterOpenBracket == FormatStyle::BAS_AlwaysBreak ||
       // as that feels like needlessly wasting whitespace, e.g.:
       //
       //   caaaaaaaaaaaall(
       //       caaaaaaaaaaaall(
       //           caaaaaaaaaaaall(
       //               caaaaaaaaaaaaaaaaaaaaaaall(aaaaaaaaaaaaaa, aaaaaaaaa))));
       Current.FakeLParens.size() > 0 &&
       Current.FakeLParens.back() > prec::Unknown) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
   }
-  if (Previous.is(TT_TemplateString) && Previous.opensScope())
+  if (Previous.is(TT_TemplateString) && Previous.opensScope()) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
+  }
 
   // Align following lines within parentheses / brackets if configured.
   // Note: This doesn't apply to macro expansion lines, which are MACRO( , , )
   // with args as children of the '(' and ',' tokens. It does not make sense to
   // align the commas with the opening paren.
   if (Style.AlignAfterOpenBracket != FormatStyle::BAS_DontAlign &&
       !CurrentState.IsCSharpGenericTypeConstraint && Previous.opensScope() &&
       Previous.isNot(TT_ObjCMethodExpr) && Previous.isNot(TT_RequiresClause) &&
       !(Current.MacroParent && Previous.MacroParent) &&
       (Current.isNot(TT_LineComment) || Previous.is(BK_BracedInit))) {
     CurrentState.Indent = State.Column + Spaces;
     CurrentState.IsAligned = true;
   }
-  if (CurrentState.AvoidBinPacking && startsNextParameter(Current, Style))
+  if (CurrentState.AvoidBinPacking && startsNextParameter(Current, Style)) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
+  }
   if (startsSegmentOfBuilderTypeCall(Current) &&
       State.Column > getNewLineColumn(State)) {
     CurrentState.ContainsUnwrappedBuilder = true;
   }
 
-  if (Current.is(TT_LambdaArrow) && Style.Language == FormatStyle::LK_Java)
+  if (Current.is(TT_LambdaArrow) && Style.Language == FormatStyle::LK_Java) {
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
+  }
   if (Current.isMemberAccess() && Previous.is(tok::r_paren) &&
       (Previous.MatchingParen &&
        (Previous.TotalLength - Previous.MatchingParen->TotalLength > 10))) {
     // If there is a function call with long parameters, break before trailing
     // calls. This prevents things like:
     //   EXPECT_CALL(SomeLongParameter).Times(
     //       2);
     // We don't want to do this for short parameters as they can just be
     // indexes.
+    DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreak");
     CurrentState.NoLineBreak = true;
   }
 
   // Don't allow the RHS of an operator to be split over multiple lines unless
   // there is a line-break right after the operator.
   // Exclude relational operators, as there, it is always more desirable to
   // have the LHS 'left' of the RHS.
   const FormatToken *P = Current.getPreviousNonComment();
if (!Current.is(tok::comment) && P &&
     // always a nice vertical separation between them and the extra line break
     // does not help.
     bool HasTwoOperands =
         P->OperatorIndex == 0 && !P->NextOperator && !P->is(TT_ConditionalExpr);
     if ((!BreakBeforeOperator &&
          !(HasTwoOperands &&
            Style.AlignOperands != FormatStyle::OAS_DontAlign)) ||
         (!CurrentState.LastOperatorWrapped && BreakBeforeOperator)) {
+      DEBUG_FORMAT_TRACE_TOKEN(Current, "NoLineBreakInOperand");
       CurrentState.NoLineBreakInOperand = true;
     }
   }
 
   State.Column += Spaces;
   if (Current.isNot(tok::comment) && Previous.is(tok::l_paren) &&
       Previous.Previous &&
       (Previous.Previous->is(tok::kw_for) || Previous.Previous->isIf())) {
for (const auto &PrecedenceLevel : llvm::reverse(Current.FakeLParens)) {
     NewParenState.Tok = nullptr;
     NewParenState.ContainsLineBreak = false;
     NewParenState.LastOperatorWrapped = true;
     NewParenState.IsChainedConditional = false;
     NewParenState.IsWrappedConditional = false;
     NewParenState.UnindentOperator = false;
     NewParenState.NoLineBreak =
         NewParenState.NoLineBreak || CurrentState.NoLineBreakInOperand;
+    DEBUG_FORMAT_TRACE_TOKEN(Current, (NewParenState.NoLineBreak ? "" : "!")
+                                          << "NoLineBreak");
 
     // Don't propagate AvoidBinPacking into subexpressions of arg/param lists.
     if (PrecedenceLevel > prec::Comma)
       NewParenState.AvoidBinPacking = false;
 
     // Indent from 'LastSpace' unless these are fake parentheses encapsulating
     // a builder type call after 'return' or, if the alignment after opening
     // brackets is disabled.

clang/lib/Format/FormatDebug.h

+//===--- FormatDebug.h - Format C++ code ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file declares utilities used in clang-format in debug mode.
+///
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_LIB_FORMAT_FORMATDEBUG_H
+#define LLVM_CLANG_LIB_FORMAT_FORMATDEBUG_H
+
+#include "llvm/Support/Regex.h"
+
+namespace clang {
+namespace format {
+namespace internal {
+
+#ifndef NDEBUG
+
+extern const llvm::Regex &DebugTraceToken();
+extern const llvm::Regex &DebugForceMustBreak();
+extern const llvm::Regex &DebugForceMustNotBreak();
+
+#define DEBUG_FORMAT_TRACE_TOKEN(Tok, Debug)                                   \
+  if (internal::DebugTraceToken().match((Tok).TokenText)) {                    \
+    llvm::dbgs() << __FILE__ << ":" << __LINE__ << ": tracing "                \
+                 << (Tok).TokenText << ": " << Debug << "\n";                  \
+  } else {                                                                     \
+  }
+
+#endif
+
+} // namespace internal
+} // namespace format
+} // namespace clang
+
+#endif
 No newline at end of file

clang/lib/Format/FormatDebug.cpp

+//===--- FormatDebug.cpp - Format C++ code ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements utilities used in clang-format in debug mode.
+///
+//===----------------------------------------------------------------------===//
+
+#include "FormatDebug.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Regex.h"
+
+namespace clang {
+namespace format {
+namespace internal {
+
+#ifndef NDEBUG
+
+namespace {
+
+llvm::cl::opt<std::string>
+    DebugTraceTokenRe("debug-trace-token",
+                      llvm::cl::desc("Regular expression to trace formatting "
+                                     "decisions of tokens whose text matches."),
+                      llvm::cl::Hidden);
+
+llvm::cl::opt<std::string> DebugForceMustBreakRe(
+    "debug-force-must-break",
+    llvm::cl::desc("Regular expression to force line breaks before tokens "
+                   "whose text matches."),
+    llvm::cl::Hidden);
+
+llvm::cl::opt<std::string> DebugForceMustNotBreakRe(
+    "debug-force-must-not-break",
+    llvm::cl::desc("Regular expression to force not breaking before tokens "
+                   "whose text matches."),
+    llvm::cl::Hidden);
+
+} // namespace
+
+const llvm::Regex &DebugTraceToken() {
+  static llvm::Regex R(DebugTraceTokenRe);
+  return R;
+}
+
+const llvm::Regex &DebugForceMustBreak() {
+  static llvm::Regex R(DebugForceMustBreakRe);
+  return R;
+}
+
+const llvm::Regex &DebugForceMustNotBreak() {
+  static llvm::Regex R(DebugForceMustNotBreakRe);
+  return R;
+}
+
+#endif
+
+} // namespace internal
+} // namespace format
+} // namespace clang
 No newline at end of file

clang/lib/Format/TokenAnnotator.cpp

 //===--- TokenAnnotator.cpp - Format C++ code -----------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
 /// This file implements a token annotator, i.e. creates
 /// \c AnnotatedTokens out of \c FormatTokens with required extra information.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "TokenAnnotator.h"
+#include "ContinuationIndenter.h"
+#include "FormatDebug.h"
 #include "FormatToken.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "format-token-annotator"
 
if (Current->is(TT_LineComment)) {
       }
     } else if (Current->SpacesRequiredBefore == 0 &&
                spaceRequiredBefore(Line, *Current)) {
       Current->SpacesRequiredBefore = 1;
     }
 
     const auto &Children = Prev->Children;
     if (!Children.empty() && Children.back()->Last->is(TT_LineComment)) {
+      DEBUG_FORMAT_TRACE_TOKEN(*Current, "MustBreakBefore");
       Current->MustBreakBefore = true;
     } else {
       Current->MustBreakBefore =
           Current->MustBreakBefore || mustBreakBefore(Line, *Current);
       if (!Current->MustBreakBefore && InFunctionDecl &&
           Current->is(TT_FunctionDeclarationName)) {
         Current->MustBreakBefore = mustBreakForReturnType(Line);
       }
+      DEBUG_FORMAT_TRACE_TOKEN(*Current, (Current->MustBreakBefore ? "" : "!")
+                                             << "MustBreakBefore");
     }
 
     Current->CanBreakBefore =
         Current->MustBreakBefore || canBreakBefore(Line, *Current);
     unsigned ChildSize = 0;
     if (Prev->Children.size() == 1) {
       FormatToken &LastOfChild = *Prev->Children[0]->Last;
       ChildSize = LastOfChild.isTrailingComment() ? Style.ColumnLimit

clang/lib/Format/UnwrappedLineFormatter.cpp

 //===--- UnwrappedLineFormatter.cpp - Format C++ code ---------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "UnwrappedLineFormatter.h"
+#include "ContinuationIndenter.h"
+#include "FormatDebug.h"
 #include "FormatToken.h"
 #include "NamespaceEndCommentsFixer.h"
 #include "WhitespaceManager.h"
 #include "llvm/Support/Debug.h"
 #include <queue>
 
 #define DEBUG_TYPE "format-formatter"
 
private:
   }
 
   /// Add the following state to the analysis queue \c Queue.
   ///
   /// Assume the current state is \p PreviousNode and has been reached with a
   /// penalty of \p Penalty. Insert a line break if \p NewLine is \c true.
   void addNextStateToQueue(unsigned Penalty, StateNode *PreviousNode,
                            bool NewLine, unsigned *Count, QueueType *Queue) {
-    if (NewLine && !Indenter->canBreak(PreviousNode->State))
+    bool MustBreak = false;
+    bool MustNotBreak = false;
+    LLVM_DEBUG({
+      if (internal::DebugForceMustBreak().match(
+              PreviousNode->State.NextToken->TokenText)) {
+        MustBreak = true;
+      }
+      if (internal::DebugForceMustNotBreak().match(
+              PreviousNode->State.NextToken->TokenText)) {
+        MustNotBreak = true;
+      }
+    });
+
+    if (NewLine && (!Indenter->canBreak(PreviousNode->State) || MustNotBreak) &&
+        !MustBreak) {
+      DEBUG_FORMAT_TRACE_TOKEN(*PreviousNode->State.NextToken,
+                               "Cannot break before");
       return;
-    if (!NewLine && Indenter->mustBreak(PreviousNode->State))
+    }
+    if (!NewLine && (Indenter->mustBreak(PreviousNode->State) || MustBreak) &&
+        !MustNotBreak) {
+      DEBUG_FORMAT_TRACE_TOKEN(*PreviousNode->State.NextToken,
+                               "Must break before");
       return;
+    }
 
     StateNode *Node = new (Allocator.Allocate())
         StateNode(PreviousNode->State, NewLine, PreviousNode);
     if (!formatChildren(Node->State, NewLine, /*DryRun=*/true, Penalty))
       return;
 
-    Penalty += Indenter->addTokenToState(Node->State, NewLine, true);
+    unsigned AddPenalty = Indenter->addTokenToState(Node->State, NewLine, true);
+    DEBUG_FORMAT_TRACE_TOKEN(*PreviousNode->State.NextToken,
+                             "Penalty for " << (!NewLine ? "not " : "")
+                                            << "breaking before token is "
+                                            << AddPenalty);
+    Penalty += AddPenalty;
 
     Queue->push(QueueItem(OrderedPenalty(Penalty, *Count), Node));
     ++(*Count);
   }
 
   /// Applies the best formatting by reconstructing the path in the
   /// solution space that leads to \c Best.
   void reconstructPath(LineState &State, StateNode *Best) {