DRAFT [clang-format] Format raw string proto text literals

[krasimir]

Summary

This is an initial draft adding formatting inside proto text raw strings.

[krasimir]

• Remove debugging macro

[djasper]

include/clang/Format/Format.h

1276

I think what we'd want here is a map from delimiter (also fix spelling) to language.

lib/Format/FormatToken.h

271

I would think that we should set the token types correctly. Fundamentally, this shouldn't be very different from how we handle JavaScript template strings. There we set the token type of the introducers to TT_TemplateString.

lib/Format/FormatTokenLexer.cpp

244

Don't use a regex here (or in the configuration). Matching regular expressions against all raw string literals in a file can be quite expensive. The string has to exactly start with a certain token sequence, so use StringRef::startsWith().

[krasimir]

Superseded by D35943.

include/clang/Format/Format.h

 
   /// \brief Format compatible with this standard, e.g. use ``A<A<int> >``
   /// instead of ``A<A<int>>`` for ``LS_Cpp03``.
   LanguageStandard Standard;
 
   /// \brief The number of columns used for tab stops.
   unsigned TabWidth;
 
+  /// \brief A regular expression matching the delimiter of a raw string
+  /// containing textual protocol buffer messages, for example ``(pb|proto)``.
+  /// The matching is case-insensitive.
+  std::string TextProtoRawStringDelimeter;
+
   /// \brief Different ways to use tab in formatting.
   enum UseTabStyle {
     /// Never use tab.
     UT_Never,
     /// Use tabs only for indentation.
     UT_ForIndentation,
     /// Use tabs only for line continuation and indentation.
     UT_ForContinuationAndIndentation,
return AccessModifierOffset == R.AccessModifierOffset &&
            SpaceInEmptyParentheses == R.SpaceInEmptyParentheses &&
            SpacesBeforeTrailingComments == R.SpacesBeforeTrailingComments &&
            SpacesInAngles == R.SpacesInAngles &&
            SpacesInContainerLiterals == R.SpacesInContainerLiterals &&
            SpacesInCStyleCastParentheses == R.SpacesInCStyleCastParentheses &&
            SpacesInParentheses == R.SpacesInParentheses &&
            SpacesInSquareBrackets == R.SpacesInSquareBrackets &&
            Standard == R.Standard && TabWidth == R.TabWidth &&
+           TextProtoRawStringDelimeter == R.TextProtoRawStringDelimeter &&
            UseTab == R.UseTab;
   }
 };
 
 /// \brief Returns a format style complying with the LLVM coding standards:
 /// http://llvm.org/docs/CodingStandards.html.
 FormatStyle getLLVMStyle();
 

lib/Format/Format.cpp

static void mapping(IO &IO, FormatStyle &Style) {
     IO.mapOptional("SpacesInContainerLiterals",
                    Style.SpacesInContainerLiterals);
     IO.mapOptional("SpacesInCStyleCastParentheses",
                    Style.SpacesInCStyleCastParentheses);
     IO.mapOptional("SpacesInParentheses", Style.SpacesInParentheses);
     IO.mapOptional("SpacesInSquareBrackets", Style.SpacesInSquareBrackets);
     IO.mapOptional("Standard", Style.Standard);
     IO.mapOptional("TabWidth", Style.TabWidth);
+    IO.mapOptional("TextProtoRawStringDelimeter",
+                   Style.TextProtoRawStringDelimeter);
     IO.mapOptional("UseTab", Style.UseTab);
   }
 };
 
 template <> struct MappingTraits<FormatStyle::BraceWrappingFlags> {
   static void mapping(IO &IO, FormatStyle::BraceWrappingFlags &Wrapping) {
     IO.mapOptional("AfterClass", Wrapping.AfterClass);
     IO.mapOptional("AfterControlStatement", Wrapping.AfterControlStatement);
LLVMStyle.IncludeCategories = {{"^\"(llvm|llvm-c|clang|clang-c)/", 2},
                                  {".*", 1}};
   LLVMStyle.IncludeIsMainRegex = "$";
   LLVMStyle.IndentCaseLabels = false;
   LLVMStyle.IndentWrappedFunctionNames = false;
   LLVMStyle.IndentWidth = 2;
   LLVMStyle.JavaScriptQuotes = FormatStyle::JSQS_Leave;
   LLVMStyle.JavaScriptWrapImports = true;
   LLVMStyle.TabWidth = 8;
+  LLVMStyle.TextProtoRawStringDelimeter = "(pb|proto)";
   LLVMStyle.MaxEmptyLinesToKeep = 1;
   LLVMStyle.KeepEmptyLinesAtTheStartOfBlocks = true;
   LLVMStyle.NamespaceIndentation = FormatStyle::NI_None;
   LLVMStyle.ObjCBlockIndentWidth = 2;
   LLVMStyle.ObjCSpaceAfterProperty = false;
   LLVMStyle.ObjCSpaceBeforeProtocolList = true;
   LLVMStyle.PointerAlignment = FormatStyle::PAS_Right;
   LLVMStyle.SpacesBeforeTrailingComments = 1;

lib/Format/FormatToken.h

struct FormatToken {
   /// Only set if \c Type == \c TT_StartOfName.
   bool PartOfMultiVariableDeclStmt = false;
 
   /// \brief Does this line comment continue a line comment section?
   ///
   /// Only set to true if \c Type == \c TT_LineComment.
   bool ContinuesLineCommentSection = false;
 
+  /// \brief Does this token start a raw string literal with proto message text.
+  bool StartsRawStringProtoLiteral = false;
+
   /// \brief If this is a bracket, this points to the matching one.
   FormatToken *MatchingParen = nullptr;
 
   /// \brief The previous token in the unwrapped line.
   FormatToken *Previous = nullptr;
 
   /// \brief The next token in the unwrapped line.
   FormatToken *Next = nullptr;

lib/Format/FormatTokenLexer.h

private:
   // an opening ${. It also maintains a stack of lexing contexts to handle
   // nested template parts by balancing curly braces.
   void handleTemplateStrings();
 
   bool tryMerge_TMacro();
 
   bool tryMergeConflictMarkers();
 
+  void tryParseTextProtoRawStringLiteral();
+
   FormatToken *getStashedToken();
 
   FormatToken *getNextToken();
 
   FormatToken *FormatTok;
   bool IsFirstToken;
   std::stack<LexerState> StateStack;
   unsigned Column;
private:
   unsigned FirstInLineIndex;
   SmallVector<FormatToken *, 16> Tokens;
   SmallVector<IdentifierInfo *, 8> ForEachMacros;
 
   bool FormattingDisabled;
 
   llvm::Regex MacroBlockBeginRegex;
   llvm::Regex MacroBlockEndRegex;
+  llvm::Regex TextProtoRawStringDelimeterRegex;
 
   void readRawToken(FormatToken &Tok);
 
   void resetLexer(unsigned Offset);
 };
 
 } // namespace format
 } // namespace clang
 
 #endif

lib/Format/FormatTokenLexer.cpp

 
 #include "FormatTokenLexer.h"
 #include "FormatToken.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Format/Format.h"
 #include "llvm/Support/Regex.h"
 
+#define DEBUG_TYPE "format-lexer"
+
+#define DBG(A) DEBUG({ llvm::dbgs() << __func__ << ":" << __LINE__ << "::" \
+  << #A << " = " << A << "\n"; });
+
 namespace clang {
 namespace format {
 
 FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
                                    const FormatStyle &Style,
                                    encoding::Encoding Encoding)
     : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
       Column(0), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
       Style(Style), IdentTable(getFormattingLangOpts(Style)),
       Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
       FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
-      MacroBlockEndRegex(Style.MacroBlockEnd) {
+      MacroBlockEndRegex(Style.MacroBlockEnd),
+      TextProtoRawStringDelimeterRegex(Style.TextProtoRawStringDelimeter,
+                                       llvm::Regex::IgnoreCase) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
                       getFormattingLangOpts(Style)));
   Lex->SetKeepWhitespaceMode(true);
 
   for (const std::string &ForEachMacro : Style.ForEachMacros)
     ForEachMacros.push_back(&IdentTable.get(ForEachMacro));
   std::sort(ForEachMacros.begin(), ForEachMacros.end());
 }
 
 ArrayRef<FormatToken *> FormatTokenLexer::lex() {
   assert(Tokens.empty());
   assert(FirstInLineIndex == 0);
   do {
     Tokens.push_back(getNextToken());
     if (Style.Language == FormatStyle::LK_JavaScript) {
       tryParseJSRegexLiteral();
       handleTemplateStrings();
     }
     tryMergePreviousTokens();
     if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
       FirstInLineIndex = Tokens.size() - 1;
+    if (Style.isCpp())
+      tryParseTextProtoRawStringLiteral();
   } while (Tokens.back()->Tok.isNot(tok::eof));
   return Tokens;
 }
 
 void FormatTokenLexer::tryMergePreviousTokens() {
   if (tryMerge_TMacro())
     return;
   if (tryMergeConflictMarkers())
void FormatTokenLexer::tryParseJSRegexLiteral() {
   // Treat regex literals like other string_literals.
   RegexToken->Tok.setKind(tok::string_literal);
   RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
   RegexToken->ColumnWidth = RegexToken->TokenText.size();
 
   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
 }
 
+// Matches the text of a raw string literal, like:
+// R"delimiter(contents)delimiter"
+// The first group captures the delimiter; the second captures the contents.
+static llvm::Regex
+    kRawStringLiteralPattern(R"regex(^R"([a-zA-Z0-9_]*)\((.*)\)\1"$)regex");
+
+void FormatTokenLexer::tryParseTextProtoRawStringLiteral() {
+  FormatToken *RawTok = Tokens.back();
+  SmallVector<StringRef, 2> Groups;
+  if (!RawTok->is(tok::string_literal))
+    return;
+  StringRef TokenText = RawTok->TokenText;
+  DBG(TokenText);
+  if (!kRawStringLiteralPattern.match(RawTok->TokenText, &Groups))
+    return;
+  assert(Groups.size() > 1);
+  StringRef Delimiter = Groups[1];
+  if (!TextProtoRawStringDelimeterRegex.match(Delimiter))
+    return;
+  DBG(Delimiter);
+  // The opening text is 'R"delimeter('.
+  StringRef OpenText = TokenText.substr(0, Delimiter.size() + 3);
+  // The closing text is ')delimeter"'.
+  StringRef CloseText =
+      TokenText.substr(TokenText.size() - Delimiter.size() - 2);
+  RawTok->TokenText = OpenText;
+  RawTok->ColumnWidth = OpenText.size();
+  RawTok->IsMultiline = false;
+  RawTok->StartsRawStringProtoLiteral = true;
+  RawTok->Tok.setKind(tok::l_brace);
+  RawTok->Tok.setIdentifierInfo(nullptr);
+  const char *Offset = Lex->getBufferLocation();
+  resetLexer(SourceMgr.getFileOffset(
+      RawTok->Tok.getLocation().getLocWithOffset(OpenText.size())));
+  while (FormatToken *Tok = getNextToken()) {
+    if (Lex->getBufferLocation() > Offset - CloseText.size()) {
+      Tok->TokenText = CloseText;
+      Tok->ColumnWidth = CloseText.size();
+      Tok->Tok.setKind(tok::r_brace);
+      Tokens.push_back(Tok);
+      break;
+    }
+    Tokens.push_back(Tok);
+  }
+  resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
+}
+
 void FormatTokenLexer::handleTemplateStrings() {
   FormatToken *BacktickToken = Tokens.back();
 
   if (BacktickToken->is(tok::l_brace)) {
     StateStack.push(LexerState::NORMAL);
     return;
   }
   if (BacktickToken->is(tok::r_brace)) {

lib/Format/UnwrappedLineParser.cpp

 
 #include "UnwrappedLineParser.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "format-parser"
 
+#define DBG(A) DEBUG({ llvm::dbgs() << __func__ << ":" << __LINE__ << "::" \
+  << #A << " = " << A << "\n"; });
+
 namespace clang {
 namespace format {
 
 class FormatTokenSource {
 public:
   virtual ~FormatTokenSource() {}
   virtual FormatToken *getNextToken() = 0;
 
private:
 
   FormatToken *Token;
 };
 
 } // end anonymous namespace
 
 class ScopedLineState {
 public:
-  ScopedLineState(UnwrappedLineParser &Parser,
+  ScopedLineState(UnwrappedLineParser &Parser, bool AddScope = true,
                   bool SwitchToPreprocessorLines = false)
-      : Parser(Parser), OriginalLines(Parser.CurrentLines) {
+      : Parser(Parser), AddScope(AddScope), OriginalLines(Parser.CurrentLines) {
+    if (!AddScope) return;
     if (SwitchToPreprocessorLines)
       Parser.CurrentLines = &Parser.PreprocessorDirectives;
     else if (!Parser.Line->Tokens.empty())
       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
     PreBlockLine = std::move(Parser.Line);
     Parser.Line = llvm::make_unique<UnwrappedLine>();
     Parser.Line->Level = PreBlockLine->Level;
     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
   }
 
   ~ScopedLineState() {
-    if (!Parser.Line->Tokens.empty()) {
+    if (!AddScope) return;
+    if (!Parser.Line->Tokens.empty())
       Parser.addUnwrappedLine();
-    }
     assert(Parser.Line->Tokens.empty());
     Parser.Line = std::move(PreBlockLine);
     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
       Parser.MustBreakBeforeNextToken = true;
     Parser.CurrentLines = OriginalLines;
   }
 
 private:
   UnwrappedLineParser &Parser;
+  const bool AddScope;
 
   std::unique_ptr<UnwrappedLine> PreBlockLine;
   SmallVectorImpl<UnwrappedLine> *OriginalLines;
 };
 
 class CompoundStatementIndenter {
 public:
   CompoundStatementIndenter(UnwrappedLineParser *Parser,
bool UnwrappedLineParser::tryToParseBracedList() {
   if (FormatTok->BlockKind == BK_Block)
     return false;
   parseBracedList();
   return true;
 }
 
 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
   bool HasError = false;
+  bool AddScope = FormatTok->StartsRawStringProtoLiteral;
+  DBG(FormatTok->TokenText);
+  DBG(AddScope);
   nextToken();
-
+  {
+    ScopedLineState State(*this, AddScope);
     // FIXME: Once we have an expression parser in the UnwrappedLineParser,
     // replace this by using parseAssigmentExpression() inside.
-  do {
+    while (!eof()) {
       if (Style.Language == FormatStyle::LK_JavaScript) {
         if (FormatTok->is(Keywords.kw_function) ||
-          FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
+            FormatTok->startsSequence(Keywords.kw_async,
+                                      Keywords.kw_function)) {
           tryToParseJSFunction();
           continue;
         }
         if (FormatTok->is(TT_JsFatArrow)) {
           nextToken();
           // Fat arrows can be followed by simple expressions or by child blocks
           // in curly braces.
           if (FormatTok->is(tok::l_brace)) {
             parseChildBlock();
             continue;
           }
         }
         if (FormatTok->is(tok::l_brace)) {
           // Could be a method inside of a braced list `{a() { return 1; }}`.
           if (tryToParseBracedList())
             continue;
           parseChildBlock();
         }
       }
+      if (FormatTok->is(tok::r_brace)) break;
       switch (FormatTok->Tok.getKind()) {
       case tok::caret:
         nextToken();
         if (FormatTok->is(tok::l_brace)) {
           parseChildBlock();
         }
         break;
       case tok::l_square:
         tryToParseLambda();
         break;
       case tok::l_paren:
         parseParens();
         // JavaScript can just have free standing methods and getters/setters in
         // object literals. Detect them by a "{" following ")".
         if (Style.Language == FormatStyle::LK_JavaScript) {
           if (FormatTok->is(tok::l_brace))
             parseChildBlock();
           break;
         }
         break;
       case tok::l_brace:
-      // Assume there are no blocks inside a braced init list apart
-      // from the ones we explicitly parse out (like lambdas).
+        // Assume there are no blocks inside a braced init list apart from the
+        // ones we explicitly parse out (like lambdas).
         FormatTok->BlockKind = BK_BracedInit;
         parseBracedList();
         break;
       case tok::r_brace:
+        assert(0);
         nextToken();
         return !HasError;
       case tok::semi:
-      // JavaScript (or more precisely TypeScript) can have semicolons in braced
-      // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be
-      // used for error recovery if we have otherwise determined that this is
-      // a braced list.
+        // JavaScript (or more precisely TypeScript) can have semicolons in
+        // braced lists (in so-called TypeMemberLists). Thus, the semicolon
+        // cannot be used for error recovery if we have otherwise determined
+        // that this is a braced list.
         if (Style.Language == FormatStyle::LK_JavaScript) {
           nextToken();
           break;
         }
         HasError = true;
         if (!ContinueOnSemicolons)
           return !HasError;
         nextToken();
         break;
       case tok::comma:
         nextToken();
         break;
       default:
         nextToken();
         break;
       }
-  } while (!eof());
-  return false;
+    }
+  } // destroy ScopedLineState
+  if (FormatTok->is(tok::r_brace)) nextToken();
+  return !HasError;
 }
 
 void UnwrappedLineParser::parseParens() {
   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
   nextToken();
   do {
     switch (FormatTok->Tok.getKind()) {
     case tok::l_paren:
do {
     assert(FormatTok);
     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
       distributeComments(Comments, FormatTok);
       Comments.clear();
       // If there is an unfinished unwrapped line, we flush the preprocessor
       // directives only after that unwrapped line was finished later.
       bool SwitchToPreprocessorLines = !Line->Tokens.empty();
-      ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
+      ScopedLineState BlockState(*this, /*AddScope=*/true,
+                                 SwitchToPreprocessorLines);
       // Comments stored before the preprocessor directive need to be output
       // before the preprocessor directive, at the same level as the
       // preprocessor directive, as we consider them to apply to the directive.
       flushComments(isOnNewLine(*FormatTok));
       parsePPDirective();
     }
     while (FormatTok->Type == TT_ConflictStart ||
            FormatTok->Type == TT_ConflictEnd ||

unittests/Format/FormatTest.cpp

 }
 
 TEST_F(FormatTest, UTF8CharacterLiteralCpp11) {
   // u8'a' is a C++17 feature, utf8 literal character, LS_Cpp11 covers
   // all modes, including C++11, C++14 and C++17
   EXPECT_EQ("auto c = u8'a';", format("auto c = u8'a';"));
 }
 
+TEST_F(FormatTest, TODO) {
+  std::string Code = R"code(auto a = R"pb(
+  field_a: OK
+  field_b: "OK"
+  field_c: "OK"
+  msg_field: {field_d: 123} )pb";)code";
+  EXPECT_EQ(Code, format(Code));
+}
+
 } // end namespace
 } // end namespace format
 } // end namespace clang