[clang-format] Support text proto messages

[krasimir]

Summary

This patch adds support for textual protocol buffer messages.

[krasimir]

• Remove newline

[krasimir]

Tests for <>-style message fields are missing because I discovered that they don't really work in a multiline setting in proto options anyways. I'll address this problem separately.

[krasimir]

• Add initial support for <>-style message fields
• Added single-line tests
• Added multiline message proto tests

[krasimir]

• Wrap-up this patch

[krasimir]

@djasper: I think this is ready for review.

[djasper]

lib/Format/ContinuationIndenter.cpp

69

Maybe rename to opensProtoMessageField() and add:

if (!LessTok.is(tok::less))
  return false;

95

Do we need to set this explicitly here? Is it not enough to set FormatStyle.BinPackParameters to false?

lib/Format/TokenAnnotator.cpp

647

Missing space...

lib/Format/UnwrappedLineParser.cpp

1355

Why not just call nextToken before calling this function instead of adding the "StartInside" parameter?

[krasimir]

• Address review comments

lib/Format/ContinuationIndenter.cpp

95

BinPackParameters doesn't work at global scope. We set these explicitly to deal with the message formatting at global scope.

lib/Format/TokenAnnotator.cpp

647

I don't understand? This has been formatted with clang-format.

include/clang/Format/Format.h

enum LanguageKind {
     /// Should be used for JavaScript.
     LK_JavaScript,
     /// Should be used for Objective-C, Objective-C++.
     LK_ObjC,
     /// Should be used for Protocol Buffers
     /// (https://developers.google.com/protocol-buffers/).
     LK_Proto,
     /// Should be used for TableGen code.
-    LK_TableGen
+    LK_TableGen,
+    /// Should be used for Protocol Buffer messages in text format
+    /// (https://developers.google.com/protocol-buffers/).
+    LK_TextProto
   };
   bool isCpp() const { return Language == LK_Cpp || Language == LK_ObjC; }
 
   /// \brief Language, this format style is targeted at.
   LanguageKind Language;
 
   /// \brief A regular expression matching macros that start a block.
   /// \code

lib/Format/ContinuationIndenter.cpp

return Previous.is(tok::comma) && !Current.isTrailingComment() &&
            Style.BreakConstructorInitializers !=
                FormatStyle::BCIS_BeforeComma) &&
           (Previous.isNot(TT_InheritanceComma) ||
            !Style.BreakBeforeInheritanceComma));
 }
 
 ContinuationIndenter::ContinuationIndenter(const FormatStyle &Style,
                                            const AdditionalKeywords &Keywords,
                                            const SourceManager &SourceMgr,
                                            WhitespaceManager &Whitespaces,
                                            encoding::Encoding Encoding,
                                            bool BinPackInconclusiveFunctions)
     : Style(Style), Keywords(Keywords), SourceMgr(SourceMgr),
       Whitespaces(Whitespaces), Encoding(Encoding),
       BinPackInconclusiveFunctions(BinPackInconclusiveFunctions),
       CommentPragmasRegex(Style.CommentPragmas) {}
 
State.Stack.push_back(ParenState(FirstIndent, FirstIndent,
                                    /*AvoidBinPacking=*/false,
                                    /*NoLineBreak=*/false));
   State.LineContainsContinuedForLoopSection = false;
   State.StartOfStringLiteral = 0;
   State.StartOfLineLevel = 0;
   State.LowestLevelOnLine = 0;
   State.IgnoreStackForComparison = false;
 
+  if (Style.Language == FormatStyle::LK_TextProto) {
+    State.Stack.back().AvoidBinPacking = true;
+    State.Stack.back().BreakBeforeParameter = true;
+  }
+
   // The first token has already been indented and thus consumed.
   moveStateToNextToken(State, DryRun, /*Newline=*/false);
   return State;
 }
 
 bool ContinuationIndenter::canBreak(const LineState &State) {
   const FormatToken &Current = *State.NextToken;
   const FormatToken &Previous = *Current.Previous;

lib/Format/Format.cpp

 template <> struct ScalarEnumerationTraits<FormatStyle::LanguageKind> {
   static void enumeration(IO &IO, FormatStyle::LanguageKind &Value) {
     IO.enumCase(Value, "Cpp", FormatStyle::LK_Cpp);
     IO.enumCase(Value, "Java", FormatStyle::LK_Java);
     IO.enumCase(Value, "JavaScript", FormatStyle::LK_JavaScript);
     IO.enumCase(Value, "ObjC", FormatStyle::LK_ObjC);
     IO.enumCase(Value, "Proto", FormatStyle::LK_Proto);
     IO.enumCase(Value, "TableGen", FormatStyle::LK_TableGen);
+    IO.enumCase(Value, "TextProto", FormatStyle::LK_TextProto);
   }
 };
 
 template <> struct ScalarEnumerationTraits<FormatStyle::LanguageStandard> {
   static void enumeration(IO &IO, FormatStyle::LanguageStandard &Value) {
     IO.enumCase(Value, "Cpp03", FormatStyle::LS_Cpp03);
     IO.enumCase(Value, "C++03", FormatStyle::LS_Cpp03);
     IO.enumCase(Value, "Cpp11", FormatStyle::LS_Cpp11);
FormatStyle getLLVMStyle() {
 
   LLVMStyle.DisableFormat = false;
   LLVMStyle.SortIncludes = true;
 
   return LLVMStyle;
 }
 
 FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
+  if (Language == FormatStyle::LK_TextProto) {
+    FormatStyle GoogleStyle = getGoogleStyle(FormatStyle::LK_Proto);
+    GoogleStyle.Language = FormatStyle::LK_TextProto;
+    return GoogleStyle;
+  }
+
   FormatStyle GoogleStyle = getLLVMStyle();
   GoogleStyle.Language = Language;
 
   GoogleStyle.AccessModifierOffset = -1;
   GoogleStyle.AlignEscapedNewlines = FormatStyle::ENAS_Left;
   GoogleStyle.AllowShortIfStatementsOnASingleLine = true;
   GoogleStyle.AllowShortLoopsOnASingleLine = true;
   GoogleStyle.AlwaysBreakBeforeMultilineStrings = true;

lib/Format/TokenAnnotator.cpp

if (CurrentToken) {
         }
         if (CurrentToken->isOneOf(tok::r_paren, tok::r_square))
           return false;
         updateParameterCount(Left, CurrentToken);
         if (CurrentToken->isOneOf(tok::colon, tok::l_brace)) {
           FormatToken *Previous = CurrentToken->getPreviousNonComment();
           if (((CurrentToken->is(tok::colon) &&
                 (!Contexts.back().ColonIsDictLiteral || !Style.isCpp())) ||
-               Style.Language == FormatStyle::LK_Proto) &&
+               Style.Language == FormatStyle::LK_Proto ||
+               Style.Language == FormatStyle::LK_TextProto) &&
               (Previous->Tok.getIdentifierInfo() ||
                Previous->is(tok::string_literal)))
             Previous->Type = TT_SelectorName;
           if (CurrentToken->is(tok::colon) ||
               Style.Language == FormatStyle::LK_JavaScript)
             Left->Type = TT_DictLiteral;
         }
         if (CurrentToken->is(tok::comma) &&
case tok::colon:
             Contexts.back().ContextKind == tok::l_square || // array type
             (Contexts.size() == 1 &&
              Line.MustBeDeclaration)) { // method/property declaration
           Tok->Type = TT_JsTypeColon;
           break;
         }
       }
       if (Contexts.back().ColonIsDictLiteral ||
-          Style.Language == FormatStyle::LK_Proto) {
+          Style.Language == FormatStyle::LK_Proto ||
+          Style.Language == FormatStyle::LK_TextProto) {
         Tok->Type = TT_DictLiteral;
+        if (Style.Language == FormatStyle::LK_TextProto) {
+          if (FormatToken *Previous = Tok->getPreviousNonComment())
+            Previous->Type = TT_SelectorName;
+        }
       } else if (Contexts.back().ColonIsObjCMethodExpr ||
                  Line.startsWith(TT_ObjCMethodSpecifier)) {
         Tok->Type = TT_ObjCMethodExpr;
         const FormatToken *BeforePrevious = Tok->Previous->Previous;
         if (!BeforePrevious ||
             !(BeforePrevious->is(TT_CastRParen) ||
               (BeforePrevious->is(TT_ObjCMethodExpr) &&
                BeforePrevious->is(tok::colon))) ||
case tok::l_paren:
                                    TT_LeadingJavaAnnotation)))
         Line.MightBeFunctionDecl = true;
       break;
     case tok::l_square:
       if (!parseSquare())
         return false;
       break;
     case tok::l_brace:
+      if (Style.Language == FormatStyle::LK_TextProto) {
+        FormatToken *Previous =Tok->getPreviousNonComment();
+        if (Previous && Previous->Type != TT_DictLiteral)
+          Previous->Type = TT_SelectorName;
+      }
       if (!parseBrace())
         return false;
       break;
     case tok::less:
       if (parseAngle()) {
         Tok->Type = TT_TemplateOpener;
       } else {
         Tok->Type = TT_BinaryOperator;
         NonTemplateLess.insert(Tok);
         CurrentToken = Tok;
         next();
       }
       break;
     case tok::r_paren:
     case tok::r_square:
       return false;

 bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
                                          const FormatToken &Right) {
   const FormatToken &Left = *Right.Previous;
   if (Right.Tok.getIdentifierInfo() && Left.Tok.getIdentifierInfo())
     return true; // Never ever merge two identifiers.
   if (Style.isCpp()) {
     if (Left.is(tok::kw_operator))
       return Right.is(tok::coloncolon);
-  } else if (Style.Language == FormatStyle::LK_Proto) {
+  } else if (Style.Language == FormatStyle::LK_Proto ||
+             Style.Language == FormatStyle::LK_TextProto) {
     if (Right.is(tok::period) &&
         Left.isOneOf(Keywords.kw_optional, Keywords.kw_required,
                      Keywords.kw_repeated, Keywords.kw_extend))
       return true;
     if (Right.is(tok::l_paren) &&
         Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
       return true;
   } else if (Style.Language == FormatStyle::LK_JavaScript) {

lib/Format/UnwrappedLineParser.h

private:
   void parsePPIf(bool IfDef);
   void parsePPElIf();
   void parsePPElse();
   void parsePPEndIf();
   void parsePPUnknown();
   void readTokenWithJavaScriptASI();
   void parseStructuralElement();
   bool tryToParseBracedList();
-  bool parseBracedList(bool ContinueOnSemicolons = false);
+  bool parseBracedList(bool ContinueOnSemicolons = false, bool StartInside = false);
   void parseParens();
   void parseSquare();
   void parseIfThenElse();
   void parseTryCatch();
   void parseForOrWhileLoop();
   void parseDoWhile();
   void parseLabel();
   void parseCaseLabel();

lib/Format/UnwrappedLineParser.cpp

 
 void UnwrappedLineParser::parseFile() {
   // The top-level context in a file always has declarations, except for pre-
   // processor directives and JavaScript files.
   bool MustBeDeclaration =
       !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript;
   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
                                           MustBeDeclaration);
+  if (Style.Language == FormatStyle::LK_TextProto)
+    parseBracedList(/*ContinueOnSemicolon=*/false, /*StartInside=*/true);
+  else
     parseLevel(/*HasOpeningBrace=*/false);
   // Make sure to format the remaining tokens.
   flushComments(true);
   addUnwrappedLine();
 }
 
 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
   bool SwitchLabelEncountered = false;
   do {
if (FormatTok->BlockKind == BK_Unknown)
     calculateBraceTypes();
   assert(FormatTok->BlockKind != BK_Unknown);
   if (FormatTok->BlockKind == BK_Block)
     return false;
   parseBracedList();
   return true;
 }
 
-bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
+bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons,
+                                          bool StartInside) {
   bool HasError = false;
-  nextToken();
+  if (!StartInside) nextToken();
 
   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
   // replace this by using parseAssigmentExpression() inside.
   do {
     if (Style.Language == FormatStyle::LK_JavaScript) {
       if (FormatTok->is(Keywords.kw_function) ||
           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
         tryToParseJSFunction();

unittests/Format/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_clang_unittest(FormatTests
   CleanupTest.cpp
   FormatTest.cpp
   FormatTestComments.cpp
   FormatTestJS.cpp
   FormatTestJava.cpp
   FormatTestObjC.cpp
   FormatTestProto.cpp
   FormatTestSelective.cpp
+  FormatTestTextProto.cpp
   NamespaceEndCommentsFixerTest.cpp
   SortImportsTestJS.cpp
   SortIncludesTest.cpp
   )
 
 target_link_libraries(FormatTests
   clangBasic
   clangFormat
   clangFrontend
   clangRewrite
   clangToolingCore
   )

unittests/Format/FormatTestTextProto.cpp

+//===- unittest/Format/FormatTestProto.cpp --------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "FormatTestUtils.h"
+#include "clang/Format/Format.h"
+#include "llvm/Support/Debug.h"
+#include "gtest/gtest.h"
+
+#define DEBUG_TYPE "format-test"
+
+namespace clang {
+namespace format {
+
+class FormatTestTextProto : public ::testing::Test {
+protected:
+  static std::string format(llvm::StringRef Code, unsigned Offset,
+                            unsigned Length, const FormatStyle &Style) {
+    DEBUG(llvm::errs() << "---\n");
+    DEBUG(llvm::errs() << Code << "\n\n");
+    std::vector<tooling::Range> Ranges(1, tooling::Range(Offset, Length));
+    tooling::Replacements Replaces = reformat(Style, Code, Ranges);
+    auto Result = applyAllReplacements(Code, Replaces);
+    EXPECT_TRUE(static_cast<bool>(Result));
+    DEBUG(llvm::errs() << "\n" << *Result << "\n\n");
+    return *Result;
+  }
+
+  static std::string format(llvm::StringRef Code) {
+    FormatStyle Style = getGoogleStyle(FormatStyle::LK_TextProto);
+    Style.ColumnLimit = 60; // To make writing tests easier.
+    return format(Code, 0, Code.size(), Style);
+  }
+
+  static void verifyFormat(llvm::StringRef Code) {
+    EXPECT_EQ(Code.str(), format(test::messUp(Code)));
+  }
+};
+
+TEST_F(FormatTestTextProto, KeepsTopLevelEntriesFittingALine) {
+  verifyFormat("field_a: OK field_b: OK field_c: OK field_d: OK field_e: OK");
+}
+
+TEST_F(FormatTestTextProto, SupportsMessageFields) {
+  verifyFormat("msg_field: {}");
+
+  verifyFormat("msg_field: {field_a: A}");
+
+  verifyFormat("msg_field: {field_a: \"OK\" field_b: 123}");
+
+  verifyFormat("msg_field: {\n"
+               "  field_a: 1\n"
+               "  field_b: OK\n"
+               "  field_c: \"OK\"\n"
+               "  field_d: 123\n"
+               "  field_e: 23\n"
+               "}");
+
+  verifyFormat("msg_field{}");
+
+  verifyFormat("msg_field{field_a: A}");
+
+  verifyFormat("msg_field{field_a: \"OK\" field_b: 123}");
+
+  verifyFormat("msg_field{\n"
+               "  field_a: 1\n"
+               "  field_b: OK\n"
+               "  field_c: \"OK\"\n"
+               "  field_d: 123\n"
+               "  field_e: 23.0\n"
+               "  field_f: false\n"
+               "  field_g: 'lala'\n"
+               "  field_h: 1234.567e-89\n"
+               "}");
+
+  verifyFormat("msg_field: {msg_field{field_a: 1}}");
+
+  verifyFormat("id: \"ala.bala\"\n"
+               "item{type: ITEM_A rank: 1 score: 90.0}\n"
+               "item{type: ITEM_B rank: 2 score: 70.5}\n"
+               "item{\n"
+               "  type: ITEM_A\n"
+               "  rank: 3\n"
+               "  score: 20.0\n"
+               "  description: \"the third item has a description\"\n"
+               "}");
+}
+
+TEST_F(FormatTestTextProto, AvoidsTopLevelBinPacking) {
+  verifyFormat("field_a: OK\n"
+               "field_b: OK\n"
+               "field_c: OK\n"
+               "field_d: OK\n"
+               "field_e: OK\n"
+               "field_f: OK");
+
+  verifyFormat("field_a: OK\n"
+               "field_b: \"OK\"\n"
+               "field_c: \"OK\"\n"
+               "msg_field: {field_d: 123}\n"
+               "field_e: OK\n"
+               "field_f: OK");
+
+  verifyFormat("field_a: OK\n"
+               "field_b: \"OK\"\n"
+               "field_c: \"OK\"\n"
+               "msg_field: {field_d: 123 field_e: OK}");
+}
+
+TEST_F(FormatTestTextProto, AddsNewlinesAfterTrailingComments) {
+  verifyFormat("field_a: OK  // Comment\n"
+               "field_b: 1");
+
+  verifyFormat("field_a: OK\n"
+               "msg_field: {\n"
+               "  field_b: OK  // Comment\n"
+               "}");
+
+  verifyFormat("field_a: OK\n"
+               "msg_field{\n"
+               "  field_b: OK  // Comment\n"
+               "}");
+}
+
+} // end namespace tooling
+} // end namespace clang