[Format] Include AnnotatedLine.Level in FormatToken.NestingLevel field

[berenm]

Summary

This makes NestingLevel more properly model the global nesting level of a
token, instead of the nesting level local to the current annotated line.

The NestingLevel field can then be used to determine the start and end of
token alignment sequences more precisely.

[berenm]

This patch might help clarifying the NestingLevel usage and can simplify the alignment computations in https://reviews.llvm.org/D21279

lib/Format/TokenAnnotator.cpp

void resetTokenMetadata(FormatToken *Token) {
     CurrentToken->Role.reset();
     CurrentToken->MatchingParen = nullptr;
     CurrentToken->FakeLParens.clear();
     CurrentToken->FakeRParens = 0;
   }
 
   void next() {
     if (CurrentToken) {
-      CurrentToken->NestingLevel = Contexts.size() - 1;
+      CurrentToken->NestingLevel = Line.Level + Contexts.size() - 1;
+
+      // Put the closing brace / paren at the same NestingLevel as the open
+      // brace / paren. We could do this as well for square and angles, but
+      // it doesn't look it goes nicely with the way the formatting is usually
+      // done
+      if (CurrentToken->Tok.isOneOf(tok::r_brace, tok::r_paren) &&
+          Contexts.size() > 1)
+        CurrentToken->NestingLevel--;
+
       CurrentToken->BindingStrength = Contexts.back().BindingStrength;
       modifyContext(*CurrentToken);
       determineTokenType(*CurrentToken);
       CurrentToken = CurrentToken->Next;
     }
 
     resetTokenMetadata(CurrentToken);
   }
if (!Current.is(TT_Unknown))
       // The token type is already known.
       return;
 
     // Line.MightBeFunctionDecl can only be true after the parentheses of a
     // function declaration have been found. In this case, 'Current' is a
     // trailing token of this declaration and thus cannot be a name.
     if (Current.is(Keywords.kw_instanceof)) {
       Current.Type = TT_BinaryOperator;
-    } else if (isStartOfName(Current) &&
-               (!Line.MightBeFunctionDecl || Current.NestingLevel != 0)) {
+    } else if (isStartOfName(Current) && (!Line.MightBeFunctionDecl ||
+                                          Current.NestingLevel != Line.Level)) {
       Contexts.back().FirstStartOfName = &Current;
       Current.Type = TT_StartOfName;
     } else if (Current.isOneOf(tok::kw_auto, tok::kw___auto_type)) {
       AutoFound = true;
     } else if (Current.is(tok::arrow) &&
                Style.Language == FormatStyle::LK_Java) {
       Current.Type = TT_LambdaArrow;
     } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration &&
-               Current.NestingLevel == 0) {
+               Current.NestingLevel == Line.Level) {
       Current.Type = TT_TrailingReturnArrow;
     } else if (Current.isOneOf(tok::star, tok::amp, tok::ampamp)) {
       Current.Type =
           determineStarAmpUsage(Current, Contexts.back().CanBeExpression &&
                                              Contexts.back().IsExpression,
                                 Contexts.back().InTemplateArgument);
     } else if (Current.isOneOf(tok::minus, tok::plus, tok::caret)) {
       Current.Type = determinePlusMinusCaretUsage(Current);
static bool isFunctionDeclarationName(const FormatToken &Current,
 
   // Find parentheses of parameter list.
   const FormatToken *Next = Current.Next;
   if (Current.is(tok::kw_operator)) {
     if (Current.Previous && Current.Previous->is(tok::coloncolon))
       return false;
     Next = skipOperatorName(Next);
   } else {
-    if (!Current.is(TT_StartOfName) || Current.NestingLevel != 0)
+    if (!Current.is(TT_StartOfName) || Current.NestingLevel != Line.Level)
       return false;
     for (; Next; Next = Next->Next) {
       if (Next->is(TT_TemplateOpener)) {
         Next = Next->MatchingParen;
       } else if (Next->is(tok::coloncolon)) {
         Next = Next->Next;
         if (!Next)
           return false;
unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
   if (Left.is(tok::semi))
     return 0;
 
   if (Style.Language == FormatStyle::LK_Java) {
     if (Right.isOneOf(Keywords.kw_extends, Keywords.kw_throws))
       return 1;
     if (Right.is(Keywords.kw_implements))
       return 2;
-    if (Left.is(tok::comma) && Left.NestingLevel == 0)
+    if (Left.is(tok::comma) && Left.NestingLevel == Line.Level)
       return 3;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {
     if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
       return 100;
     if (Left.is(TT_JsTypeColon))
       return 35;
     if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
         (Right.is(TT_TemplateString) && Right.TokenText.startswith("}")))
unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
   }
 
   if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
       Right.is(tok::kw_operator)) {
     if (Line.startsWith(tok::kw_for) && Right.PartOfMultiVariableDeclStmt)
       return 3;
     if (Left.is(TT_StartOfName))
       return 110;
-    if (InFunctionDecl && Right.NestingLevel == 0)
+    if (InFunctionDecl && Right.NestingLevel == Line.Level)
       return Style.PenaltyReturnTypeOnItsOwnLine;
     return 200;
   }
   if (Right.is(TT_PointerOrReference))
     return 190;
   if (Right.is(TT_LambdaArrow))
     return 110;
   if (Left.is(tok::equal) && Right.is(tok::l_brace))
if (Left.is(tok::l_brace) && Line.Level == 0 &&
       // JavaScript top-level enum key/value pairs are put on separate lines
       // instead of bin-packing.
       return true;
     if (Right.is(tok::r_brace) && Left.is(tok::l_brace) &&
         !Left.Children.empty())
       // Support AllowShortFunctionsOnASingleLine for JavaScript.
       return Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_None ||
              Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_Empty ||
-             (Left.NestingLevel == 0 && Line.Level == 0 &&
+             (Left.NestingLevel == Line.Level && Line.Level == 0 &&
               Style.AllowShortFunctionsOnASingleLine ==
                   FormatStyle::SFS_Inline);
   } else if (Style.Language == FormatStyle::LK_Java) {
     if (Right.is(tok::plus) && Left.is(tok::string_literal) && Right.Next &&
         Right.Next->is(tok::string_literal))
       return true;
   }
 
bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   if (Right.Previous->IsUnterminatedLiteral)
     return true;
   if (Right.is(tok::lessless) && Right.Next &&
       Right.Previous->is(tok::string_literal) &&
       Right.Next->is(tok::string_literal))
     return true;
   if (Right.Previous->ClosesTemplateDeclaration &&
       Right.Previous->MatchingParen &&
-      Right.Previous->MatchingParen->NestingLevel == 0 &&
+      Right.Previous->MatchingParen->NestingLevel == Line.Level &&
       Style.AlwaysBreakTemplateDeclarations)
     return true;
   if ((Right.isOneOf(TT_CtorInitializerComma, TT_CtorInitializerColon)) &&
       Style.BreakConstructorInitializersBeforeComma &&
       !Style.ConstructorInitializerAllOnOneLineOrOnePerLine)
     return true;
   if (Right.is(tok::string_literal) && Right.TokenText.startswith("R\""))
     // Raw string literals are special wrt. line breaks. The author has made a
     // deliberate choice and might have aligned the contents of the string
     // literal accordingly. Thus, we try keep existing line breaks.
     return Right.NewlinesBefore > 0;
-  if (Right.Previous->is(tok::l_brace) && Right.NestingLevel == 1 &&
+  if (Right.Previous->is(tok::l_brace) &&
+      Right.NestingLevel == Line.Level + 1 &&
       Style.Language == FormatStyle::LK_Proto)
     // Don't put enums onto single lines in protocol buffers.
     return true;
   if (Right.is(TT_InlineASMBrace))
     return Right.HasUnescapedNewline;
   if (isAllmanBrace(Left) || isAllmanBrace(Right))
     return (Line.startsWith(tok::kw_enum) && Style.BraceWrapping.AfterEnum) ||
            (Line.startsWith(tok::kw_class) && Style.BraceWrapping.AfterClass) ||
if (Left.isOneOf(tok::kw_return, tok::kw_continue, tok::kw_break,
         (NonComment &&
          NonComment->isOneOf(tok::kw_return, tok::kw_continue, tok::kw_break,
                              tok::kw_throw)))
       return false; // Otherwise a semicolon is inserted.
     if (Left.is(TT_JsFatArrow) && Right.is(tok::l_brace))
       return false;
     if (Left.is(TT_JsTypeColon))
       return true;
-    if (Right.NestingLevel == 0 && Right.is(Keywords.kw_is))
+    if (Right.NestingLevel == Line.Level && Right.is(Keywords.kw_is))
       return false;
     if (Left.is(Keywords.kw_in))
       return Style.BreakBeforeBinaryOperators == FormatStyle::BOS_None;
     if (Right.is(Keywords.kw_in))
       return Style.BreakBeforeBinaryOperators != FormatStyle::BOS_None;
     if (Right.is(Keywords.kw_as))
       return false; // must not break before as in 'x as type' casts
   }
bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
   if (Left.is(TT_RangeBasedForLoopColon))
     return true;
   if (Right.is(TT_RangeBasedForLoopColon))
     return false;
   if (Left.isOneOf(TT_TemplateCloser, TT_UnaryOperator) ||
       Left.is(tok::kw_operator))
     return false;
   if (Left.is(tok::equal) && !Right.isOneOf(tok::kw_default, tok::kw_delete) &&
-      Line.Type == LT_VirtualFunctionDecl && Left.NestingLevel == 0)
+      Line.Type == LT_VirtualFunctionDecl && Left.NestingLevel == Line.Level)
     return false;
   if (Left.is(tok::l_paren) && Left.is(TT_AttributeParen))
     return false;
   if (Left.is(tok::l_paren) && Left.Previous &&
       (Left.Previous->isOneOf(TT_BinaryOperator, TT_CastRParen)))
     return false;
   if (Right.is(TT_ImplicitStringLiteral))
     return false;