[clang-format] Reorganize RawStringFormat based on language

[krasimir]

Summary

This patch changes the structure for raw string formatting options by making it
language based (enumerate delimiters per language) as opposed to delimiter-based
(specify the language for a delimiter). The raw string formatting now uses an
appropriate style from the .clang-format file, if exists.

[krasimir]

• Update comments

[bkramer]

lg

include/clang/Format/Format.h

enum PointerAlignmentStyle {
     PAS_Middle
   };
 
   /// \brief Pointer and reference alignment style.
   PointerAlignmentStyle PointerAlignment;
 
   /// See documentation of ``RawStringFormats``.
   struct RawStringFormat {
-    /// \brief The delimiter that this raw string format matches.
-    std::string Delimiter;
     /// \brief The language of this raw string.
     LanguageKind Language;
+    /// \brief A list of raw string delimiters that match this language.
+    std::vector<std::string> Delimiters;
     /// \brief The style name on which this raw string format is based on.
     /// If not specified, the raw string format is based on the style that this
     /// format is based on.
     std::string BasedOnStyle;
     bool operator==(const RawStringFormat &Other) const {
-      return Delimiter == Other.Delimiter && Language == Other.Language &&
+      return Language == Other.Language && Delimiters == Other.Delimiters &&
              BasedOnStyle == Other.BasedOnStyle;
     }
   };
 
   /// \brief Raw string delimiters denoting that the raw string contents are
   /// code in a particular language and can be reformatted.
   ///
   /// A raw string with a matching delimiter will be reformatted assuming the
   /// specified language based on a predefined style given by 'BasedOnStyle'.
   /// If 'BasedOnStyle' is not found, the formatting is based on llvm style.
+  /// \brief Defines hints for detecting supported languages code blocks in raw
+  /// strings.
+  ///
+  /// A raw string with a matching delimiter will be reformatted assuming the
+  /// specified language based on the style for that language defined in the
+  /// .clang-format file. If no style has been defined in the .clang-format file
+  /// for the specific language, a predefined style given by 'BasedOnStyle' is
+  /// used. If 'BasedOnStyle' is not found, the formatting is based on llvm
+  /// style.
   ///
   /// To configure this in the .clang-format file, use:
   /// \code{.yaml}
   ///   RawStringFormats:
-  ///     - Delimiter: 'pb'
-  ///       Language:  TextProto
-  ///       BasedOnStyle: llvm
-  ///     - Delimiter: 'proto'
-  ///       Language:  TextProto
-  ///       BasedOnStyle: google
+  ///     - Language: TextProto
+  ///         Delimiters:
+  ///           - 'pb'
+  ///           - 'proto'
+  ///         BasedOnStyle: google
+  ///     - Language: Cpp
+  ///         Delimiters:
+  ///           - 'cc'
+  ///           - 'cpp'
+  ///         BasedOnStyle: llvm
   /// \endcode
   std::vector<RawStringFormat> RawStringFormats;
 
   /// \brief If ``true``, clang-format will attempt to re-flow comments.
   /// \code
   ///    false:
   ///    // veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information
   ///    /* second veryVeryVeryVeryVeryVeryVeryVeryVeryVeryVeryLongComment with plenty of information */

lib/Format/ContinuationIndenter.cpp

static llvm::Optional<StringRef> getRawStringDelimiter(StringRef TokenText) {
   if (!TokenText.substr(RParenPos + 1).startswith(Delimiter))
     return None;
   return Delimiter;
 }
 
 RawStringFormatStyleManager::RawStringFormatStyleManager(
     const FormatStyle &CodeStyle) {
   for (const auto &RawStringFormat : CodeStyle.RawStringFormats) {
-    FormatStyle Style;
+    for (StringRef Delimiter : RawStringFormat.Delimiters) {
+      llvm::Optional<FormatStyle> LanguageStyle =
+          CodeStyle.GetLanguageStyle(RawStringFormat.Language);
+      if (!LanguageStyle) {
+        FormatStyle PredefinedStyle;
         if (!getPredefinedStyle(RawStringFormat.BasedOnStyle,
-                            RawStringFormat.Language, &Style)) {
-      Style = getLLVMStyle();
-      Style.Language = RawStringFormat.Language;
+                                RawStringFormat.Language, &PredefinedStyle)) {
+          PredefinedStyle = getLLVMStyle();
+          PredefinedStyle.Language = RawStringFormat.Language;
+        }
+        LanguageStyle = PredefinedStyle;
+      }
+      LanguageStyle->ColumnLimit = CodeStyle.ColumnLimit;
+      DelimiterStyle.insert({Delimiter, *LanguageStyle});
     }
-    Style.ColumnLimit = CodeStyle.ColumnLimit;
-    DelimiterStyle.insert({RawStringFormat.Delimiter, Style});
   }
 }
 
 llvm::Optional<FormatStyle>
 RawStringFormatStyleManager::get(StringRef Delimiter) const {
   auto It = DelimiterStyle.find(Delimiter);
   if (It == DelimiterStyle.end())
     return None;

lib/Format/Format.cpp

static void enumeration(IO &IO, FormatStyle::IncludeBlocksStyle &Value) {
     IO.enumCase(Value, "Preserve", FormatStyle::IBS_Preserve);
     IO.enumCase(Value, "Merge", FormatStyle::IBS_Merge);
     IO.enumCase(Value, "Regroup", FormatStyle::IBS_Regroup);
   }
 };
 
 template <> struct MappingTraits<FormatStyle::RawStringFormat> {
   static void mapping(IO &IO, FormatStyle::RawStringFormat &Format) {
-    IO.mapOptional("Delimiter", Format.Delimiter);
     IO.mapOptional("Language", Format.Language);
+    IO.mapOptional("Delimiters", Format.Delimiters);
     IO.mapOptional("BasedOnStyle", Format.BasedOnStyle);
   }
 };
 
 // Allows to read vector<FormatStyle> while keeping default values.
 // IO.getContext() should contain a pointer to the FormatStyle structure, that
 // will be used to get default values for missing keys.
 // If the first element has no Language specified, it will be treated as the
FormatStyle getLLVMStyle() {
   LLVMStyle.NamespaceIndentation = FormatStyle::NI_None;
   LLVMStyle.ObjCBlockIndentWidth = 2;
   LLVMStyle.ObjCSpaceAfterProperty = false;
   LLVMStyle.ObjCSpaceBeforeProtocolList = true;
   LLVMStyle.PointerAlignment = FormatStyle::PAS_Right;
   LLVMStyle.SpacesBeforeTrailingComments = 1;
   LLVMStyle.Standard = FormatStyle::LS_Cpp11;
   LLVMStyle.UseTab = FormatStyle::UT_Never;
-  LLVMStyle.RawStringFormats = {{"pb", FormatStyle::LK_TextProto, "google"}};
   LLVMStyle.ReflowComments = true;
   LLVMStyle.SpacesInParentheses = false;
   LLVMStyle.SpacesInSquareBrackets = false;
   LLVMStyle.SpaceInEmptyParentheses = false;
   LLVMStyle.SpacesInContainerLiterals = true;
   LLVMStyle.SpacesInCStyleCastParentheses = false;
   LLVMStyle.SpaceAfterCStyleCast = false;
   LLVMStyle.SpaceAfterTemplateKeyword = true;
FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
   GoogleStyle.IncludeCategories = {
       {"^<ext/.*\\.h>", 2}, {"^<.*\\.h>", 1}, {"^<.*", 2}, {".*", 3}};
   GoogleStyle.IncludeIsMainRegex = "([-_](test|unittest))?$";
   GoogleStyle.IndentCaseLabels = true;
   GoogleStyle.KeepEmptyLinesAtTheStartOfBlocks = false;
   GoogleStyle.ObjCSpaceAfterProperty = false;
   GoogleStyle.ObjCSpaceBeforeProtocolList = false;
   GoogleStyle.PointerAlignment = FormatStyle::PAS_Left;
+  GoogleStyle.RawStringFormats = {{
+      FormatStyle::LK_TextProto,
+      /*Delimiters=*/
+      {
+          "pb",
+          "PB",
+          "proto",
+          "PROTO",
+          "textproto",
+          "TEXTPROTO",
+      },
+      /*BasedOnStyle=*/"google",
+  }};
   GoogleStyle.SpacesBeforeTrailingComments = 2;
   GoogleStyle.Standard = FormatStyle::LS_Auto;
 
   GoogleStyle.PenaltyReturnTypeOnItsOwnLine = 200;
   GoogleStyle.PenaltyBreakBeforeFirstCallParameter = 1;
 
   if (Language == FormatStyle::LK_Java) {
     GoogleStyle.AlignAfterOpenBracket = FormatStyle::BAS_DontAlign;

unittests/Format/FormatTest.cpp

CHECK_PARSE("IncludeCategories:\n"
               "    Priority: 2\n"
               "  - Regex: .*\n"
               "    Priority: 1",
               IncludeCategories, ExpectedCategories);
   CHECK_PARSE("IncludeIsMainRegex: 'abc$'", IncludeIsMainRegex, "abc$");
 
   Style.RawStringFormats.clear();
   std::vector<FormatStyle::RawStringFormat> ExpectedRawStringFormats = {
-      {"pb", FormatStyle::LK_TextProto, "llvm"},
-      {"cpp", FormatStyle::LK_Cpp, "google"}};
+      {FormatStyle::LK_TextProto, {"pb", "proto"}, "llvm"},
+      {FormatStyle::LK_Cpp, {"cc", "cpp"}, "google"},
+  };
 
   CHECK_PARSE("RawStringFormats:\n"
-              "  - Delimiter: 'pb'\n"
-              "    Language: TextProto\n"
+              "  - Language: TextProto\n"
+              "    Delimiters:\n"
+              "      - 'pb'\n"
+              "      - 'proto'\n"
               "    BasedOnStyle: llvm\n"
-              "  - Delimiter: 'cpp'\n"
-              "    Language: Cpp\n"
-              "    BasedOnStyle: google",
+              "  - Language: Cpp\n"
+              "    Delimiters:\n"
+              "      - 'cc'\n"
+              "      - 'cpp'\n"
+              "    BasedOnStyle: google\n",
               RawStringFormats, ExpectedRawStringFormats);
 }
 
 TEST_F(FormatTest, ParsesConfigurationWithLanguages) {
   FormatStyle Style = {};
   Style.Language = FormatStyle::LK_Cpp;
   CHECK_PARSE("Language: Cpp\n"
               "IndentWidth: 12",

unittests/Format/FormatTestRawStrings.cpp

FormatStyle getLLVMStyleWithColumns(unsigned ColumnLimit) {
     return getStyleWithColumns(getLLVMStyle(), ColumnLimit);
   }
 
   int ReplacementCount;
 
   FormatStyle getRawStringPbStyleWithColumns(unsigned ColumnLimit) {
     FormatStyle Style = getLLVMStyle();
     Style.ColumnLimit = ColumnLimit;
-    Style.RawStringFormats = {{/*Delimiter=*/"pb",
-                               /*Kind=*/FormatStyle::LK_TextProto,
+    Style.RawStringFormats = {{/*Language=*/FormatStyle::LK_TextProto,
+                               /*Delimiters=*/{"pb"},
                                /*BasedOnStyle=*/"google"}};
     return Style;
   }
 
   FormatStyle getRawStringLLVMCppStyleBasedOn(std::string BasedOnStyle) {
     FormatStyle Style = getLLVMStyle();
-    Style.RawStringFormats = {{/*Delimiter=*/"cpp",
-                               /*Kind=*/FormatStyle::LK_Cpp, BasedOnStyle}};
+    Style.RawStringFormats = {{/*Language=*/FormatStyle::LK_Cpp,
+                               /*Delimiters=*/{"cpp"}, BasedOnStyle}};
     return Style;
   }
 
   FormatStyle getRawStringGoogleCppStyleBasedOn(std::string BasedOnStyle) {
     FormatStyle Style = getGoogleStyle(FormatStyle::LK_Cpp);
-    Style.RawStringFormats = {{/*Delimiter=*/"cpp",
-                               /*Kind=*/FormatStyle::LK_Cpp, BasedOnStyle}};
+    Style.RawStringFormats = {{/*Language=*/FormatStyle::LK_Cpp,
+                               /*Delimiters=*/{"cpp"}, BasedOnStyle}};
     return Style;
   }
 
   // Gcc 4.8 doesn't support raw string literals in macros, which breaks some
   // build bots. We use this function instead.
   void expect_eq(const std::string Expected, const std::string Actual) {
     EXPECT_EQ(Expected, Actual);
   }
expect_eq(R"test(int *i = R"cpp(int* p = nullptr;)cpp")test",
                    getRawStringLLVMCppStyleBasedOn("google")));
 
   // Use the llvm style if the raw string style has no BasedOnStyle=llvm.
   expect_eq(R"test(int* i = R"cpp(int *p = nullptr;)cpp")test",
             format(R"test(int * i = R"cpp(int * p = nullptr;)cpp")test",
                    getRawStringGoogleCppStyleBasedOn("llvm")));
 }
 
+TEST_F(FormatTestRawStrings, UsesConfigurationOverBaseStyle) {
+  // llvm style puts '*' on the right.
+  // google style puts '*' on the left.
+
+  // Uses the configured google style inside raw strings even if BasedOnStyle in
+  // the raw string format is llvm.
+  FormatStyle Style = getGoogleStyle(FormatStyle::LK_Cpp);
+  EXPECT_EQ(0, parseConfiguration("---\n"
+                                  "Language: Cpp\n"
+                                  "BasedOnStyle: Google", &Style).value());
+  Style.RawStringFormats = {{FormatStyle::LK_Cpp, {"cpp"}, "llvm"}};
+  expect_eq(R"test(int* i = R"cpp(int* j = 0;)cpp";)test",
+            format(R"test(int * i = R"cpp(int * j = 0;)cpp";)test", Style));
+}
+
 TEST_F(FormatTestRawStrings, MatchesDelimitersCaseSensitively) {
   // Don't touch the 'PB' raw string, format the 'pb' raw string.
   expect_eq(R"test(
 s = R"PB(item:1)PB";
 t = R"pb(item: 1)pb";)test",
             format(R"test(
 s = R"PB(item:1)PB";
 t = R"pb(item:1)pb";)test",
                    getRawStringPbStyleWithColumns(40)));
-
-  FormatStyle MixedStyle = getLLVMStyle();
-  MixedStyle.RawStringFormats = {
-      {/*Delimiter=*/"cpp", /*Kind=*/FormatStyle::LK_Cpp,
-       /*BasedOnStyle=*/"llvm"},
-      {/*Delimiter=*/"CPP", /*Kind=*/FormatStyle::LK_Cpp,
-       /*BasedOnStyle=*/"google"}};
-
-  // Format the 'cpp' raw string with '*' on the right.
-  // Format the 'CPP' raw string with '*' on the left.
-  // Do not format the 'Cpp' raw string.
-  // Do not format non-raw strings.
-  expect_eq(R"test(
-a = R"cpp(int *i = 0;)cpp";
-b = R"CPP(int* j = 0;)CPP";
-c = R"Cpp(int * k = 0;)Cpp";
-d = R"cpp(int * k = 0;)Cpp";)test",
-            format(R"test(
-a = R"cpp(int * i = 0;)cpp";
-b = R"CPP(int * j = 0;)CPP";
-c = R"Cpp(int * k = 0;)Cpp";
-d = R"cpp(int * k = 0;)Cpp";)test",
-                   MixedStyle));
 }
 
 TEST_F(FormatTestRawStrings, ReformatsShortRawStringsOnSingleLine) {
   expect_eq(
       R"test(P p = TP(R"pb()pb");)test",
       format(
           R"test(P p = TP(R"pb( )pb");)test",
           getRawStringPbStyleWithColumns(40)));