[clang-format] Support text proto messages

[krasimir]

Summary

This patch adds support for textual protocol buffer messages.

[krasimir]

• Remove newline

[krasimir]

Tests for <>-style message fields are missing because I discovered that they don't really work in a multiline setting in proto options anyways. I'll address this problem separately.

[krasimir]

• Add initial support for <>-style message fields
• Added single-line tests
• Added multiline message proto tests

[krasimir]

• Wrap-up this patch

[krasimir]

@djasper: I think this is ready for review.

[djasper]

lib/Format/ContinuationIndenter.cpp

71 ↗

(On Diff #104646)

Maybe rename to opensProtoMessageField() and add:

if (!LessTok.is(tok::less))
  return false;

107 ↗

(On Diff #104646)

Do we need to set this explicitly here? Is it not enough to set FormatStyle.BinPackParameters to false?

lib/Format/TokenAnnotator.cpp

657 ↗

(On Diff #104646)

Missing space...

lib/Format/UnwrappedLineParser.cpp

1360 ↗

(On Diff #104646)

Why not just call nextToken before calling this function instead of adding the "StartInside" parameter?

[krasimir]

• Address review comments

lib/Format/ContinuationIndenter.cpp

107 ↗

(On Diff #104646)

BinPackParameters doesn't work at global scope. We set these explicitly to deal with the message formatting at global scope.

lib/Format/TokenAnnotator.cpp

657 ↗

(On Diff #104646)

I don't understand? This has been formatted with clang-format.

cfe/trunk/include/clang/Format/Format.h

enum LanguageKind {
     /// Should be used for JavaScript.
     LK_JavaScript,
     /// Should be used for Objective-C, Objective-C++.
     LK_ObjC,
     /// Should be used for Protocol Buffers
     /// (https://developers.google.com/protocol-buffers/).
     LK_Proto,
     /// Should be used for TableGen code.
-    LK_TableGen
+    LK_TableGen,
+    /// Should be used for Protocol Buffer messages in text format
+    /// (https://developers.google.com/protocol-buffers/).
+    LK_TextProto
   };
   bool isCpp() const { return Language == LK_Cpp || Language == LK_ObjC; }
 
   /// \brief Language, this format style is targeted at.
   LanguageKind Language;
 
   /// \brief A regular expression matching macros that start a block.
   /// \code

cfe/trunk/lib/Format/ContinuationIndenter.cpp

static bool startsNextParameter(const FormatToken &Current,
   return Previous.is(tok::comma) && !Current.isTrailingComment() &&
          ((Previous.isNot(TT_CtorInitializerComma) ||
            Style.BreakConstructorInitializers !=
                FormatStyle::BCIS_BeforeComma) &&
           (Previous.isNot(TT_InheritanceComma) ||
            !Style.BreakBeforeInheritanceComma));
 }
 
+static bool opensProtoMessageField(const FormatToken &LessTok,
+                                   const FormatStyle &Style) {
+  if (LessTok.isNot(tok::less))
+    return false;
+  return Style.Language == FormatStyle::LK_TextProto ||
+         (Style.Language == FormatStyle::LK_Proto &&
+          (LessTok.NestingLevel > 0 ||
+           (LessTok.Previous && LessTok.Previous->is(tok::equal))));
+}
+
 ContinuationIndenter::ContinuationIndenter(const FormatStyle &Style,
                                            const AdditionalKeywords &Keywords,
                                            const SourceManager &SourceMgr,
                                            WhitespaceManager &Whitespaces,
                                            encoding::Encoding Encoding,
                                            bool BinPackInconclusiveFunctions)
     : Style(Style), Keywords(Keywords), SourceMgr(SourceMgr),
       Whitespaces(Whitespaces), Encoding(Encoding),
State.Stack.push_back(ParenState(FirstIndent, FirstIndent,
                                    /*AvoidBinPacking=*/false,
                                    /*NoLineBreak=*/false));
   State.LineContainsContinuedForLoopSection = false;
   State.StartOfStringLiteral = 0;
   State.StartOfLineLevel = 0;
   State.LowestLevelOnLine = 0;
   State.IgnoreStackForComparison = false;
 
+  if (Style.Language == FormatStyle::LK_TextProto) {
+    // We need this in order to deal with the bin packing of text fields at
+    // global scope.
+    State.Stack.back().AvoidBinPacking = true;
+    State.Stack.back().BreakBeforeParameter = true;
+  }
+
   // The first token has already been indented and thus consumed.
   moveStateToNextToken(State, DryRun, /*Newline=*/false);
   return State;
 }
 
 bool ContinuationIndenter::canBreak(const LineState &State) {
   const FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *Current.Previous;
if ((startsNextParameter(Current, Style) || Previous.is(tok::semi) ||
         Previous.isNot(tok::question)) ||
        (!Style.BreakBeforeTernaryOperators &&
         Previous.is(TT_ConditionalExpr))) &&
       State.Stack.back().BreakBeforeParameter && !Current.isTrailingComment() &&
       !Current.isOneOf(tok::r_paren, tok::r_brace))
     return true;
   if (((Previous.is(TT_DictLiteral) && Previous.is(tok::l_brace)) ||
        (Previous.is(TT_ArrayInitializerLSquare) &&
-        Previous.ParameterCount > 1)) &&
+        Previous.ParameterCount > 1) ||
+       opensProtoMessageField(Previous, Style)) &&
       Style.ColumnLimit > 0 &&
       getLengthToMatchingParen(Previous) + State.Column - 1 >
           getColumnLimit(State))
     return true;
 
   const FormatToken &BreakConstructorInitializersToken =
       Style.BreakConstructorInitializers == FormatStyle::BCIS_AfterColon
           ? Previous
if (Previous.MatchingParen) {
       HasTrailingCall = Next && Next->isMemberAccess();
     }
     if (HasTrailingCall && State.Stack.size() > 1 &&
         State.Stack[State.Stack.size() - 2].CallContinuation == 0)
       State.Stack.back().LastSpace = State.Column;
   }
 }
 
-static bool lessOpensProtoMessageField(const FormatToken &LessTok,
-                                       const LineState &State) {
-  assert(LessTok.is(tok::less));
-  return LessTok.NestingLevel > 0 ||
-         (LessTok.Previous && LessTok.Previous->is(tok::equal));
-}
-
 unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
                                                  bool DryRun) {
   FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *State.NextToken->Previous;
 
   // Extra penalty that needs to be added because of the way certain line
   // breaks are chosen.
   unsigned Penalty = 0;
if (PreviousNonComment &&
           TT_LeadingJavaAnnotation) &&
       Current.isNot(TT_BinaryOperator) && !PreviousNonComment->opensScope())
     State.Stack.back().BreakBeforeParameter = true;
 
   // If we break after { or the [ of an array initializer, we should also break
   // before the corresponding } or ].
   if (PreviousNonComment &&
       (PreviousNonComment->isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
-      (Style.Language == FormatStyle::LK_Proto &&
-       PreviousNonComment->is(tok::less) &&
-       lessOpensProtoMessageField(*PreviousNonComment, State)) ||
+       opensProtoMessageField(*PreviousNonComment, Style) ||
        (PreviousNonComment->is(TT_TemplateString) &&
         PreviousNonComment->opensScope())))
     State.Stack.back().BreakBeforeClosingBrace = true;
 
   if (State.Stack.back().AvoidBinPacking) {
     // If we are breaking after '(', '{', '<', this is not bin packing
     // unless AllowAllParametersOfDeclarationOnNextLine is false or this is a
     // dict/object literal.
if (Style.Language == FormatStyle::LK_Java &&
       Current.isOneOf(Keywords.kw_implements, Keywords.kw_extends))
     return std::max(State.Stack.back().LastSpace,
                     State.Stack.back().Indent + Style.ContinuationIndentWidth);
 
   if (NextNonComment->is(tok::l_brace) && NextNonComment->BlockKind == BK_Block)
     return Current.NestingLevel == 0 ? State.FirstIndent
                                      : State.Stack.back().Indent;
   if ((Current.isOneOf(tok::r_brace, tok::r_square) ||
-       (Current.is(tok::greater) && Style.Language == FormatStyle::LK_Proto)) &&
+       (Current.is(tok::greater) &&
+        (Style.Language == FormatStyle::LK_Proto ||
+         Style.Language == FormatStyle::LK_TextProto))) &&
       State.Stack.size() > 1) {
     if (Current.closesBlockOrBlockTypeList(Style))
       return State.Stack[State.Stack.size() - 2].NestedBlockIndent;
     if (Current.MatchingParen &&
         Current.MatchingParen->BlockKind == BK_BracedInit)
       return State.Stack[State.Stack.size() - 2].LastSpace;
     return State.FirstIndent;
   }
void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
 
   unsigned NewIndent;
   unsigned LastSpace = State.Stack.back().LastSpace;
   bool AvoidBinPacking;
   bool BreakBeforeParameter = false;
   unsigned NestedBlockIndent = std::max(State.Stack.back().StartOfFunctionCall,
                                         State.Stack.back().NestedBlockIndent);
   if (Current.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) ||
-      (Style.Language == FormatStyle::LK_Proto && Current.is(tok::less) &&
-       lessOpensProtoMessageField(Current, State))) {
+      opensProtoMessageField(Current, Style)) {
     if (Current.opensBlockOrBlockTypeList(Style)) {
       NewIndent = Style.IndentWidth +
                   std::min(State.Column, State.Stack.back().NestedBlockIndent);
     } else {
       NewIndent = State.Stack.back().LastSpace + Style.ContinuationIndentWidth;
     }
     const FormatToken *NextNoComment = Current.getNextNonComment();
     bool EndsInComma = Current.MatchingParen &&
                        Current.MatchingParen->Previous &&
                        Current.MatchingParen->Previous->is(tok::comma);
     AvoidBinPacking =
         EndsInComma || Current.is(TT_DictLiteral) ||
-        Style.Language == FormatStyle::LK_Proto || !Style.BinPackArguments ||
+        Style.Language == FormatStyle::LK_Proto ||
+        Style.Language == FormatStyle::LK_TextProto ||
+        !Style.BinPackArguments ||
         (NextNoComment &&
          NextNoComment->isOneOf(TT_DesignatedInitializerPeriod,
                                 TT_DesignatedInitializerLSquare));
     BreakBeforeParameter = EndsInComma;
     if (Current.ParameterCount > 1)
       NestedBlockIndent = std::max(NestedBlockIndent, State.Column + 1);
   } else {
     NewIndent = Style.ContinuationIndentWidth +

cfe/trunk/lib/Format/Format.cpp

 template <> struct ScalarEnumerationTraits<FormatStyle::LanguageKind> {
   static void enumeration(IO &IO, FormatStyle::LanguageKind &Value) {
     IO.enumCase(Value, "Cpp", FormatStyle::LK_Cpp);
     IO.enumCase(Value, "Java", FormatStyle::LK_Java);
     IO.enumCase(Value, "JavaScript", FormatStyle::LK_JavaScript);
     IO.enumCase(Value, "ObjC", FormatStyle::LK_ObjC);
     IO.enumCase(Value, "Proto", FormatStyle::LK_Proto);
     IO.enumCase(Value, "TableGen", FormatStyle::LK_TableGen);
+    IO.enumCase(Value, "TextProto", FormatStyle::LK_TextProto);
   }
 };
 
 template <> struct ScalarEnumerationTraits<FormatStyle::LanguageStandard> {
   static void enumeration(IO &IO, FormatStyle::LanguageStandard &Value) {
     IO.enumCase(Value, "Cpp03", FormatStyle::LS_Cpp03);
     IO.enumCase(Value, "C++03", FormatStyle::LS_Cpp03);
     IO.enumCase(Value, "Cpp11", FormatStyle::LS_Cpp11);
FormatStyle getLLVMStyle() {
   LLVMStyle.DisableFormat = false;
   LLVMStyle.SortIncludes = true;
   LLVMStyle.SortUsingDeclarations = true;
 
   return LLVMStyle;
 }
 
 FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
+  if (Language == FormatStyle::LK_TextProto) {
+    FormatStyle GoogleStyle = getGoogleStyle(FormatStyle::LK_Proto);
+    GoogleStyle.Language = FormatStyle::LK_TextProto;
+    return GoogleStyle;
+  }
+
   FormatStyle GoogleStyle = getLLVMStyle();
   GoogleStyle.Language = Language;
 
   GoogleStyle.AccessModifierOffset = -1;
   GoogleStyle.AlignEscapedNewlines = FormatStyle::ENAS_Left;
   GoogleStyle.AllowShortIfStatementsOnASingleLine = true;
   GoogleStyle.AllowShortLoopsOnASingleLine = true;
   GoogleStyle.AlwaysBreakBeforeMultilineStrings = true;

cfe/trunk/lib/Format/FormatToken.h

struct FormatToken {
   /// list that should be indented with a block indent.
   bool opensBlockOrBlockTypeList(const FormatStyle &Style) const {
     if (is(TT_TemplateString) && opensScope())
       return true;
     return is(TT_ArrayInitializerLSquare) ||
            (is(tok::l_brace) &&
             (BlockKind == BK_Block || is(TT_DictLiteral) ||
              (!Style.Cpp11BracedListStyle && NestingLevel == 0))) ||
-           (is(tok::less) && Style.Language == FormatStyle::LK_Proto);
+           (is(tok::less) && (Style.Language == FormatStyle::LK_Proto ||
+                              Style.Language == FormatStyle::LK_TextProto));
   }
 
   /// \brief Same as opensBlockOrBlockTypeList, but for the closing token.
   bool closesBlockOrBlockTypeList(const FormatStyle &Style) const {
     if (is(TT_TemplateString) && closesScope())
       return true;
     return MatchingParen && MatchingParen->opensBlockOrBlockTypeList(Style);
   }

cfe/trunk/lib/Format/TokenAnnotator.cpp

while (CurrentToken) {
       }
       if (CurrentToken->is(tok::question) &&
           Style.Language == FormatStyle::LK_Java) {
         next();
         continue;
       }
       if (CurrentToken->isOneOf(tok::r_paren, tok::r_square, tok::r_brace) ||
           (CurrentToken->isOneOf(tok::colon, tok::question) && InExprContext &&
-           Style.Language != FormatStyle::LK_Proto))
+           Style.Language != FormatStyle::LK_Proto &&
+           Style.Language != FormatStyle::LK_TextProto))
         return false;
       // If a && or || is found and interpreted as a binary operator, this set
       // of angles is likely part of something like "a < b && c > d". If the
       // angles are inside an expression, the ||/&& might also be a binary
       // operator that was misinterpreted because we are parsing template
       // parameters.
       // FIXME: This is getting out of hand, write a decent parser.
       if (CurrentToken->Previous->isOneOf(tok::pipepipe, tok::ampamp) &&
if (CurrentToken) {
         }
         if (CurrentToken->isOneOf(tok::r_paren, tok::r_square))
           return false;
         updateParameterCount(Left, CurrentToken);
         if (CurrentToken->isOneOf(tok::colon, tok::l_brace, tok::less)) {
           FormatToken *Previous = CurrentToken->getPreviousNonComment();
           if (((CurrentToken->is(tok::colon) &&
                 (!Contexts.back().ColonIsDictLiteral || !Style.isCpp())) ||
-               Style.Language == FormatStyle::LK_Proto) &&
+               Style.Language == FormatStyle::LK_Proto ||
+               Style.Language == FormatStyle::LK_TextProto) &&
               (Previous->Tok.getIdentifierInfo() ||
                Previous->is(tok::string_literal)))
             Previous->Type = TT_SelectorName;
           if (CurrentToken->is(tok::colon) ||
               Style.Language == FormatStyle::LK_JavaScript)
             Left->Type = TT_DictLiteral;
         }
         if (CurrentToken->is(tok::comma) &&
case tok::colon:
             Contexts.back().ContextKind == tok::l_square || // array type
             (Contexts.size() == 1 &&
              Line.MustBeDeclaration)) { // method/property declaration
           Tok->Type = TT_JsTypeColon;
           break;
         }
       }
       if (Contexts.back().ColonIsDictLiteral ||
-          Style.Language == FormatStyle::LK_Proto) {
+          Style.Language == FormatStyle::LK_Proto ||
+          Style.Language == FormatStyle::LK_TextProto) {
         Tok->Type = TT_DictLiteral;
+        if (Style.Language == FormatStyle::LK_TextProto) {
+          if (FormatToken *Previous = Tok->getPreviousNonComment())
+            Previous->Type = TT_SelectorName;
+        }
       } else if (Contexts.back().ColonIsObjCMethodExpr ||
                  Line.startsWith(TT_ObjCMethodSpecifier)) {
         Tok->Type = TT_ObjCMethodExpr;
         const FormatToken *BeforePrevious = Tok->Previous->Previous;
         if (!BeforePrevious ||
             !(BeforePrevious->is(TT_CastRParen) ||
               (BeforePrevious->is(TT_ObjCMethodExpr) &&
                BeforePrevious->is(tok::colon))) ||
case tok::l_paren:
                                    TT_LeadingJavaAnnotation)))
         Line.MightBeFunctionDecl = true;
       break;
     case tok::l_square:
       if (!parseSquare())
         return false;
       break;
     case tok::l_brace:
+      if (Style.Language == FormatStyle::LK_TextProto) {
+        FormatToken *Previous =Tok->getPreviousNonComment();
+        if (Previous && Previous->Type != TT_DictLiteral)
+          Previous->Type = TT_SelectorName;
+      }
       if (!parseBrace())
         return false;
       break;
     case tok::less:
       if (parseAngle()) {
         Tok->Type = TT_TemplateOpener;
+        if (Style.Language == FormatStyle::LK_TextProto) {
+          FormatToken *Previous = Tok->getPreviousNonComment();
+          if (Previous && Previous->Type != TT_DictLiteral)
+            Previous->Type = TT_SelectorName;
+        }
       } else {
         Tok->Type = TT_BinaryOperator;
         NonTemplateLess.insert(Tok);
         CurrentToken = Tok;
         next();
       }
       break;
     case tok::r_paren:
private:
   /// and other tokens that we treat like binary operators.
   int getCurrentPrecedence() {
     if (Current) {
       const FormatToken *NextNonComment = Current->getNextNonComment();
       if (Current->is(TT_ConditionalExpr))
         return prec::Conditional;
       if (NextNonComment && Current->is(TT_SelectorName) &&
           (NextNonComment->is(TT_DictLiteral) ||
-           (Style.Language == FormatStyle::LK_Proto &&
+           ((Style.Language == FormatStyle::LK_Proto ||
+             Style.Language == FormatStyle::LK_TextProto) &&
             NextNonComment->is(tok::less))))
         return prec::Assignment;
       if (Current->is(TT_JsComputedPropertyName))
         return prec::Assignment;
       if (Current->is(TT_LambdaArrow))
         return prec::Comma;
       if (Current->is(TT_JsFatArrow))
         return prec::Assignment;

 bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
                                          const FormatToken &Right) {
   const FormatToken &Left = *Right.Previous;
   if (Right.Tok.getIdentifierInfo() && Left.Tok.getIdentifierInfo())
     return true; // Never ever merge two identifiers.
   if (Style.isCpp()) {
     if (Left.is(tok::kw_operator))
       return Right.is(tok::coloncolon);
-  } else if (Style.Language == FormatStyle::LK_Proto) {
+  } else if (Style.Language == FormatStyle::LK_Proto ||
+             Style.Language == FormatStyle::LK_TextProto) {
     if (Right.is(tok::period) &&
         Left.isOneOf(Keywords.kw_optional, Keywords.kw_required,
                      Keywords.kw_repeated, Keywords.kw_extend))
       return true;
     if (Right.is(tok::l_paren) &&
         Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
       return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {

cfe/trunk/lib/Format/UnwrappedLineParser.cpp

 
 void UnwrappedLineParser::parseFile() {
   // The top-level context in a file always has declarations, except for pre-
   // processor directives and JavaScript files.
   bool MustBeDeclaration =
       !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript;
   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
                                           MustBeDeclaration);
+  if (Style.Language == FormatStyle::LK_TextProto)
+    parseBracedList();
+  else
     parseLevel(/*HasOpeningBrace=*/false);
   // Make sure to format the remaining tokens.
   flushComments(true);
   addUnwrappedLine();
 }
 
 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
   bool SwitchLabelEncountered = false;
   do {
if (FormatTok->is(tok::string_literal))
       nextToken();
     addUnwrappedLine();
     return;
   }
   switch (FormatTok->Tok.getKind()) {
   case tok::at:
     nextToken();
     if (FormatTok->Tok.is(tok::l_brace)) {
+      nextToken();
       parseBracedList();
       break;
     }
     switch (FormatTok->Tok.getObjCKeywordID()) {
     case tok::objc_public:
     case tok::objc_protected:
     case tok::objc_package:
     case tok::objc_private:
void UnwrappedLineParser::parseStructuralElement() {
   default:
     break;
   }
   do {
     const FormatToken *Previous = getPreviousToken();
     switch (FormatTok->Tok.getKind()) {
     case tok::at:
       nextToken();
-      if (FormatTok->Tok.is(tok::l_brace))
+      if (FormatTok->Tok.is(tok::l_brace)) {
+        nextToken();
         parseBracedList();
+      }
       break;
     case tok::kw_enum:
       // Ignore if this is part of "template <enum ...".
       if (Previous && Previous->is(tok::less)) {
         nextToken();
         break;
       }
 
case tok::equal:
       if (FormatTok->is(TT_JsFatArrow)) {
         nextToken();
         if (FormatTok->is(tok::l_brace))
           parseChildBlock();
         break;
       }
 
       nextToken();
-      if (FormatTok->Tok.is(tok::l_brace))
+      if (FormatTok->Tok.is(tok::l_brace)) {
+        nextToken();
         parseBracedList();
-      else if (Style.Language == FormatStyle::LK_Proto &&
-               FormatTok->Tok.is(tok::less))
+      } else if (Style.Language == FormatStyle::LK_Proto &&
+               FormatTok->Tok.is(tok::less)) {
+        nextToken();
         parseBracedList(/*ContinueOnSemicolons=*/false,
                         /*ClosingBraceKind=*/tok::greater);
+      }
       break;
     case tok::l_square:
       parseSquare();
       break;
     case tok::kw_new:
       parseNew();
       break;
     default:

 }
 
 bool UnwrappedLineParser::tryToParseBracedList() {
   if (FormatTok->BlockKind == BK_Unknown)
     calculateBraceTypes();
   assert(FormatTok->BlockKind != BK_Unknown);
   if (FormatTok->BlockKind == BK_Block)
     return false;
+  nextToken();
   parseBracedList();
   return true;
 }
 
 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
                                           tok::TokenKind ClosingBraceKind) {
   bool HasError = false;
-  nextToken();
 
   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
   // replace this by using parseAssigmentExpression() inside.
   do {
     if (Style.Language == FormatStyle::LK_JavaScript) {
       if (FormatTok->is(Keywords.kw_function) ||
           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();
case tok::l_paren:
           parseChildBlock();
         break;
       }
       break;
     case tok::l_brace:
       // Assume there are no blocks inside a braced init list apart
       // from the ones we explicitly parse out (like lambdas).
       FormatTok->BlockKind = BK_BracedInit;
+      nextToken();
       parseBracedList();
       break;
     case tok::semi:
       // JavaScript (or more precisely TypeScript) can have semicolons in braced
       // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be
       // used for error recovery if we have otherwise determined that this is
       // a braced list.
       if (Style.Language == FormatStyle::LK_JavaScript) {
case tok::l_square:
       tryToParseLambda();
       break;
     case tok::l_brace:
       if (!tryToParseBracedList())
         parseChildBlock();
       break;
     case tok::at:
       nextToken();
-      if (FormatTok->Tok.is(tok::l_brace))
+      if (FormatTok->Tok.is(tok::l_brace)) {
+        nextToken();
         parseBracedList();
+      }
       break;
     case tok::kw_class:
       if (Style.Language == FormatStyle::LK_JavaScript)
         parseRecord(/*ParseAsExpr=*/true);
       else
         nextToken();
       break;
     case tok::identifier:
case tok::l_square:
       break;
     case tok::l_brace: {
       if (!tryToParseBracedList())
         parseChildBlock();
       break;
     }
     case tok::at:
       nextToken();
-      if (FormatTok->Tok.is(tok::l_brace))
+      if (FormatTok->Tok.is(tok::l_brace)) {
+        nextToken();
         parseBracedList();
+      }
       break;
     default:
       nextToken();
       break;
     }
   } while (!eof());
 }
 
if (Style.Language == FormatStyle::LK_Java) {
     return true;
   }
   if (Style.Language == FormatStyle::LK_Proto) {
     parseBlock(/*MustBeDeclaration=*/true);
     return true;
   }
 
   // Parse enum body.
+  nextToken();
   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
   if (HasError) {
     if (FormatTok->is(tok::semi))
       nextToken();
     addUnwrappedLine();
   }
   return true;
 
while (Tok) {
     }
     // FIXME: This will also mark enums with braces in the arguments to enum
     // constants as "not simple". This is probably fine in practice, though.
     Tok = Tokens->getNextToken();
   }
   FormatTok = Tokens->setPosition(StoredPosition);
 
   if (IsSimple) {
+    nextToken();
     parseBracedList();
     addUnwrappedLine();
     return;
   }
 
   // Parse the body of a more complex enum.
   // First add a line for everything up to the "{".
   nextToken();
if (FormatTok->is(tok::semi))
       return;
     if (Line->Tokens.size() == 0) {
       // Common issue: Automatic Semicolon Insertion wrapped the line, so the
       // import statement should terminate.
       return;
     }
     if (FormatTok->is(tok::l_brace)) {
       FormatTok->BlockKind = BK_Block;
+      nextToken();
       parseBracedList();
     } else {
       nextToken();
     }
   }
 }
 
 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,

cfe/trunk/unittests/Format/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_clang_unittest(FormatTests
   CleanupTest.cpp
   FormatTest.cpp
   FormatTestComments.cpp
   FormatTestJS.cpp
   FormatTestJava.cpp
   FormatTestObjC.cpp
   FormatTestProto.cpp
   FormatTestSelective.cpp
+  FormatTestTextProto.cpp
   NamespaceEndCommentsFixerTest.cpp
   SortImportsTestJS.cpp
   SortIncludesTest.cpp
   UsingDeclarationsSorterTest.cpp
   )
 
 target_link_libraries(FormatTests
   clangBasic
   clangFormat
   clangFrontend
   clangRewrite
   clangToolingCore
   )

cfe/trunk/unittests/Format/FormatTestTextProto.cpp

+//===- unittest/Format/FormatTestProto.cpp --------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "FormatTestUtils.h"
+#include "clang/Format/Format.h"
+#include "llvm/Support/Debug.h"
+#include "gtest/gtest.h"
+
+#define DEBUG_TYPE "format-test"
+
+namespace clang {
+namespace format {
+
+class FormatTestTextProto : public ::testing::Test {
+protected:
+  static std::string format(llvm::StringRef Code, unsigned Offset,
+                            unsigned Length, const FormatStyle &Style) {
+    DEBUG(llvm::errs() << "---\n");
+    DEBUG(llvm::errs() << Code << "\n\n");
+    std::vector<tooling::Range> Ranges(1, tooling::Range(Offset, Length));
+    tooling::Replacements Replaces = reformat(Style, Code, Ranges);
+    auto Result = applyAllReplacements(Code, Replaces);
+    EXPECT_TRUE(static_cast<bool>(Result));
+    DEBUG(llvm::errs() << "\n" << *Result << "\n\n");
+    return *Result;
+  }
+
+  static std::string format(llvm::StringRef Code) {
+    FormatStyle Style = getGoogleStyle(FormatStyle::LK_TextProto);
+    Style.ColumnLimit = 60; // To make writing tests easier.
+    return format(Code, 0, Code.size(), Style);
+  }
+
+  static void verifyFormat(llvm::StringRef Code) {
+    EXPECT_EQ(Code.str(), format(test::messUp(Code)));
+  }
+};
+
+TEST_F(FormatTestTextProto, KeepsTopLevelEntriesFittingALine) {
+  verifyFormat("field_a: OK field_b: OK field_c: OK field_d: OK field_e: OK");
+}
+
+TEST_F(FormatTestTextProto, SupportsMessageFields) {
+  verifyFormat("msg_field: {}");
+
+  verifyFormat("msg_field: {field_a: A}");
+
+  verifyFormat("msg_field: {field_a: \"OK\" field_b: 123}");
+
+  verifyFormat("msg_field: {\n"
+               "  field_a: 1\n"
+               "  field_b: OK\n"
+               "  field_c: \"OK\"\n"
+               "  field_d: 123\n"
+               "  field_e: 23\n"
+               "}");
+
+  verifyFormat("msg_field{}");
+
+  verifyFormat("msg_field{field_a: A}");
+
+  verifyFormat("msg_field{field_a: \"OK\" field_b: 123}");
+
+  verifyFormat("msg_field{\n"
+               "  field_a: 1\n"
+               "  field_b: OK\n"
+               "  field_c: \"OK\"\n"
+               "  field_d: 123\n"
+               "  field_e: 23.0\n"
+               "  field_f: false\n"
+               "  field_g: 'lala'\n"
+               "  field_h: 1234.567e-89\n"
+               "}");
+
+  verifyFormat("msg_field: {msg_field{field_a: 1}}");
+
+  verifyFormat("id: \"ala.bala\"\n"
+               "item{type: ITEM_A rank: 1 score: 90.0}\n"
+               "item{type: ITEM_B rank: 2 score: 70.5}\n"
+               "item{\n"
+               "  type: ITEM_A\n"
+               "  rank: 3\n"
+               "  score: 20.0\n"
+               "  description: \"the third item has a description\"\n"
+               "}");
+}
+
+TEST_F(FormatTestTextProto, AvoidsTopLevelBinPacking) {
+  verifyFormat("field_a: OK\n"
+               "field_b: OK\n"
+               "field_c: OK\n"
+               "field_d: OK\n"
+               "field_e: OK\n"
+               "field_f: OK");
+
+  verifyFormat("field_a: OK\n"
+               "field_b: \"OK\"\n"
+               "field_c: \"OK\"\n"
+               "msg_field: {field_d: 123}\n"
+               "field_e: OK\n"
+               "field_f: OK");
+
+  verifyFormat("field_a: OK\n"
+               "field_b: \"OK\"\n"
+               "field_c: \"OK\"\n"
+               "msg_field: {field_d: 123 field_e: OK}");
+
+  verifyFormat("a: {\n"
+               "  field_a: OK\n"
+               "  field_b{field_c: OK}\n"
+               "  field_d: OKOKOK\n"
+               "  field_e: OK\n"
+               "}");
+
+  verifyFormat("field_a: OK,\n"
+               "field_b{field_c: OK},\n"
+               "field_d: OKOKOK,\n"
+               "field_e: OK");
+}
+
+TEST_F(FormatTestTextProto, AddsNewlinesAfterTrailingComments) {
+  verifyFormat("field_a: OK  // Comment\n"
+               "field_b: 1");
+
+  verifyFormat("field_a: OK\n"
+               "msg_field: {\n"
+               "  field_b: OK  // Comment\n"
+               "}");
+
+  verifyFormat("field_a: OK\n"
+               "msg_field{\n"
+               "  field_b: OK  // Comment\n"
+               "}");
+}
+
+TEST_F(FormatTestTextProto, SupportsAngleBracketMessageFields) {
+  // Single-line tests
+  verifyFormat("msg_field<>");
+  verifyFormat("msg_field: <>");
+  verifyFormat("msg_field<field_a: OK>");
+  verifyFormat("msg_field: <field_a: 123>");
+  verifyFormat("msg_field<field_a<>>");
+  verifyFormat("msg_field<field_a<field_b<>>>");
+  verifyFormat("msg_field: <field_a<field_b: <>>>");
+  verifyFormat("msg_field<field_a: OK, field_b: \"OK\">");
+  verifyFormat("msg_field<field_a: OK field_b: <>, field_c: OK>");
+  verifyFormat("msg_field<field_a{field_b: 1}, field_c: <field_d: 2>>");
+  verifyFormat("msg_field: <field_a: OK, field_b: \"OK\">");
+  verifyFormat("msg_field: <field_a: OK field_b: <>, field_c: OK>");
+  verifyFormat("msg_field: <field_a{field_b: 1}, field_c: <field_d: 2>>");
+  verifyFormat("field_a: \"OK\", msg_field: <field_b: 123>, field_c: {}");
+  verifyFormat("field_a<field_b: 1>, msg_field: <field_b: 123>, field_c<>");
+  verifyFormat("field_a<field_b: 1> msg_field: <field_b: 123> field_c<>");
+  verifyFormat("field<field<field: <>>, field<>> field: <field: 1>");
+
+  // Multiple lines tests
+  verifyFormat("msg_field<\n"
+               "  field_a: OK\n"
+               "  field_b: \"OK\"\n"
+               "  field_c: 1\n"
+               "  field_d: 12.5\n"
+               "  field_e: OK\n"
+               ">");
+
+  verifyFormat("msg_field: <>\n"
+               "field_c: \"OK\",\n"
+               "msg_field: <field_d: 123>\n"
+               "field_e: OK\n"
+               "msg_field: <field_d: 12>");
+
+  verifyFormat("field_a: OK,\n"
+               "field_b<field_c: OK>,\n"
+               "field_d: <12.5>,\n"
+               "field_e: OK");
+
+  verifyFormat("field_a: OK\n"
+               "field_b<field_c: OK>\n"
+               "field_d: <12.5>\n"
+               "field_e: OKOKOK");
+
+  verifyFormat("msg_field<\n"
+               "  field_a: OK,\n"
+               "  field_b<field_c: OK>,\n"
+               "  field_d: <12.5>,\n"
+               "  field_e: OK\n"
+               ">");
+
+  verifyFormat("msg_field<\n"
+               "  field_a: <field: OK>,\n"
+               "  field_b<field_c: OK>,\n"
+               "  field_d: <12.5>,\n"
+               "  field_e: OK,\n"
+               ">");
+
+  verifyFormat("msg_field: <\n"
+               "  field_a: \"OK\"\n"
+               "  msg_field: {field_b: OK}\n"
+               "  field_g: OK\n"
+               "  field_g: OK\n"
+               "  field_g: OK\n"
+               ">");
+
+  verifyFormat("field_a{\n"
+               "  field_d: ok\n"
+               "  field_b: <field_c: 1>\n"
+               "  field_d: ok\n"
+               "  field_d: ok\n"
+               "}");
+
+  verifyFormat("field_a: {\n"
+               "  field_d: ok\n"
+               "  field_b: <field_c: 1>\n"
+               "  field_d: ok\n"
+               "  field_d: ok\n"
+               "}");
+
+  verifyFormat("field_a: <f1: 1, f2: <>>\n"
+               "field_b<\n"
+               "  field_b1: <>\n"
+               "  field_b2: ok,\n"
+               "  field_b3: <\n"
+               "    field_x{}  // Comment\n"
+               "    field_y: {field_z: 1}\n"
+               "    field_w: ok\n"
+               "  >\n"
+               "  field{\n"
+               "    field_x<>  // Comment\n"
+               "    field_y: <field_z: 1>\n"
+               "    field_w: ok\n"
+               "    msg_field: <\n"
+               "      field: <>\n"
+               "      field: <field: 1>\n"
+               "      field: <field: 2>\n"
+               "      field: <field: 3>\n"
+               "      field: <field: 4>\n"
+               "      field: ok\n"
+               "    >\n"
+               "  }\n"
+               ">\n"
+               "field: OK,\n"
+               "field_c<field<field<>>>");
+}
+
+} // end namespace tooling
+} // end namespace clang