[clang-format] Add option to group multiple #include blocks when sorting includes

[KrzysztofKapusta]

Summary

This patch allows grouping multiple #include blocks together and sort all includes as one big block.
Additionally, sorted includes can be regrouped after sorting based on configured categories.

[krasimir]

Looks good! Just a few nits:

lib/Format/Format.cpp

1570

Please replace this with two explicit checks for the expected styles. Plus, an else if with an empty body is super awkward.

unittests/Format/SortIncludesTest.cpp

357

What is this testing?

[KrzysztofKapusta]

Addressed all comments. Changed one testcase to better reflect the desired effect.

[KrzysztofKapusta]

unittests/Format/SortIncludesTest.cpp

357

I changed this to better reflect what I was trying to test. Now the expected order is not sorted from a..c

[krasimir]

Thank you! Do you have commit access?

[KrzysztofKapusta]

@krasimir thanks for the review! I don't have commit rights. Would you push this for me?

docs/ClangFormatStyleOptions.rst

 
   For example: BOOST_FOREACH.
 
+**IncludeBlocks** (``IncludeBlocksStyle``)
+  Dependent on the value, multiple ``#include`` blocks can be sorted
+  as one and divided based on category.
+
+  Possible values:
+
+  * ``IBS_Preserve`` (in configuration: ``Preserve``)
+    Sort each ``#include`` block separately.
+
+    .. code-block:: c++
+
+       #include "b.h"               into      #include "b.h"
+
+       #include <lib/main.h>                  #include "a.h"
+       #include "a.h"                         #include <lib/main.h>
+
+  * ``IBS_Merge`` (in configuration: ``Merge``)
+    Merge multiple ``#include`` blocks together and sort as one.
+
+    .. code-block:: c++
+
+       #include "b.h"               into      #include "a.h"
+                                              #include "b.h"
+       #include <lib/main.h>                  #include <lib/main.h>
+       #include "a.h"
+
+  * ``IBS_Regroup`` (in configuration: ``Regroup``)
+    Merge multiple ``#include`` blocks together and sort as one.
+    Then split into groups based on category priority. See ``IncludeCategories``.
+
+    .. code-block:: c++
+
+       #include "b.h"               into      #include "a.h"
+                                              #include "b.h"
+       #include <lib/main.h>
+       #include "a.h"                         #include <lib/main.h>
+
+
+
 **IncludeCategories** (``std::vector<IncludeCategory>``)
   Regular expressions denoting the different ``#include`` categories
   used for ordering ``#includes``.

include/clang/Format/Format.h

   /// For example: BOOST_FOREACH.
   std::vector<std::string> ForEachMacros;
 
+  /// \brief Styles for sorting multiple ``#include`` blocks.
+  enum IncludeBlocksStyle {
+    /// \brief Sort each ``#include`` block separately.
+    /// \code
+    ///    #include "b.h"               into      #include "b.h"
+    ///
+    ///    #include <lib/main.h>                  #include "a.h"
+    ///    #include "a.h"                         #include <lib/main.h>
+    /// \endcode
+    IBS_Preserve,
+    /// \brief Merge multiple ``#include`` blocks together and sort as one.
+    /// \code
+    ///    #include "b.h"               into      #include "a.h"
+    ///                                           #include "b.h"
+    ///    #include <lib/main.h>                  #include <lib/main.h>
+    ///    #include "a.h"
+    /// \endcode
+    IBS_Merge,
+    /// \brief Merge multiple ``#include`` blocks together and sort as one.
+    /// Then split into groups based on category priority. See
+    /// ``IncludeCategories``.
+    /// \code
+    ///    #include "b.h"               into      #include "a.h"
+    ///                                           #include "b.h"
+    ///    #include <lib/main.h>
+    ///    #include "a.h"                         #include <lib/main.h>
+    /// \endcode
+    IBS_Regroup,
+  };
+
+  /// \brief Dependent on the value, multiple ``#include`` blocks can be sorted
+  /// as one and divided based on category.
+  IncludeBlocksStyle IncludeBlocks;
+
   /// \brief See documentation of ``IncludeCategories``.
   struct IncludeCategory {
     /// \brief The regular expression that this category matches.

                R.ExperimentalAutoDetectBinPacking &&
            FixNamespaceComments == R.FixNamespaceComments &&
            ForEachMacros == R.ForEachMacros &&
+           IncludeBlocks == R.IncludeBlocks &&
            IncludeCategories == R.IncludeCategories &&
            IndentCaseLabels == R.IndentCaseLabels &&
            IndentPPDirectives == R.IndentPPDirectives &&

lib/Format/Format.cpp

                    Style.ExperimentalAutoDetectBinPacking);
     IO.mapOptional("FixNamespaceComments", Style.FixNamespaceComments);
     IO.mapOptional("ForEachMacros", Style.ForEachMacros);
+    IO.mapOptional("IncludeBlocks", Style.IncludeBlocks);
     IO.mapOptional("IncludeCategories", Style.IncludeCategories);
     IO.mapOptional("IncludeIsMainRegex", Style.IncludeIsMainRegex);
     IO.mapOptional("IndentCaseLabels", Style.IndentCaseLabels);

   }
 };
 
+template <> struct ScalarEnumerationTraits<FormatStyle::IncludeBlocksStyle> {
+  static void enumeration(IO &IO, FormatStyle::IncludeBlocksStyle &Value) {
+    IO.enumCase(Value, "Preserve", FormatStyle::IBS_Preserve);
+    IO.enumCase(Value, "Merge", FormatStyle::IBS_Merge);
+    IO.enumCase(Value, "Regroup", FormatStyle::IBS_Regroup);
+  }
+};
+
 template <> struct MappingTraits<FormatStyle::RawStringFormat> {
   static void mapping(IO &IO, FormatStyle::RawStringFormat &Format) {
     IO.mapOptional("Delimiter", Format.Delimiter);

                                  {"^(<|\"(gtest|gmock|isl|json)/)", 3},
                                  {".*", 1}};
   LLVMStyle.IncludeIsMainRegex = "(Test)?$";
+  LLVMStyle.IncludeBlocks = FormatStyle::IBS_Preserve;
   LLVMStyle.IndentCaseLabels = false;
   LLVMStyle.IndentPPDirectives = FormatStyle::PPDIS_None;
   LLVMStyle.IndentWrappedFunctionNames = false;

                             }),
                 Indices.end());
 
+  int CurrentCategory = Includes.front().Category;
+
   // If the #includes are out of order, we generate a single replacement fixing
   // the entire block. Otherwise, no replacement is generated.
   if (Indices.size() == Includes.size() &&
-      std::is_sorted(Indices.begin(), Indices.end()))
+      std::is_sorted(Indices.begin(), Indices.end()) &&
+      Style.IncludeBlocks == FormatStyle::IBS_Preserve)
     return;
 
   std::string result;
   for (unsigned Index : Indices) {
-    if (!result.empty())
+    if (!result.empty()) {
       result += "\n";
+      if (Style.IncludeBlocks == FormatStyle::IBS_Regroup &&
+          CurrentCategory != Includes[Index].Category)
+        result += "\n";
+    }
     result += Includes[Index].Text;
     if (Cursor && CursorIndex == Index)
       *Cursor = IncludesBeginOffset + result.size() - CursorToEOLOffset;
+    CurrentCategory = Includes[Index].Category;
   }
 
   auto Err = Replaces.add(tooling::Replacement(

     else if (Trimmed == "// clang-format on")
       FormattingOff = false;
 
+    const bool EmptyLineSkipped =
+        Trimmed.empty() && (Style.IncludeBlocks == FormatStyle::IBS_Merge ||
+                            Style.IncludeBlocks == FormatStyle::IBS_Regroup);
+
     if (!FormattingOff && !Line.endswith("\\")) {
       if (IncludeRegex.match(Line, &Matches)) {
         StringRef IncludeName = Matches[2];

         if (Category == 0)
           MainIncludeFound = true;
         IncludesInBlock.push_back({IncludeName, Line, Prev, Category});
-      } else if (!IncludesInBlock.empty()) {
+      } else if (!IncludesInBlock.empty() && !EmptyLineSkipped) {
         sortCppIncludes(Style, IncludesInBlock, Ranges, FileName, Replaces,
                         Cursor);
         IncludesInBlock.clear();

unittests/Format/SortIncludesTest.cpp

   EXPECT_TRUE(sortIncludes(Style, Code, GetCodeRange(Code), "a.cc").empty());
 }
 
+TEST_F(SortIncludesTest, SortedIncludesInMultipleBlocksAreMerged) {
+  Style.IncludeBlocks = FormatStyle::IBS_Merge;
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"b.h\"\n"
+            "#include \"c.h\"\n",
+            sort("#include \"a.h\"\n"
+                 "#include \"c.h\"\n"
+                 "\n"
+                 "\n"
+                 "#include \"b.h\"\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"b.h\"\n"
+            "#include \"c.h\"\n",
+            sort("#include \"a.h\"\n"
+                 "#include \"c.h\"\n"
+                 "\n"
+                 "\n"
+                 "#include \"b.h\"\n"));
+}
+
 TEST_F(SortIncludesTest, SupportClangFormatOff) {
   EXPECT_EQ("#include <a>\n"
             "#include <b>\n"

                  "#include \"b.h\"\n"));
 }
 
+TEST_F(SortIncludesTest, SortsAllBlocksWhenMerging) {
+  Style.IncludeBlocks = FormatStyle::IBS_Merge;
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"b.h\"\n"
+            "#include \"c.h\"\n",
+            sort("#include \"a.h\"\n"
+                 "#include \"c.h\"\n"
+                 "\n"
+                 "#include \"b.h\"\n"));
+}
+
+TEST_F(SortIncludesTest, CommentsAlwaysSeparateGroups) {
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"c.h\"\n"
+            "// comment\n"
+            "#include \"b.h\"\n",
+            sort("#include \"c.h\"\n"
+                 "#include \"a.h\"\n"
+                 "// comment\n"
+                 "#include \"b.h\"\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Merge;
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"c.h\"\n"
+            "// comment\n"
+            "#include \"b.h\"\n",
+            sort("#include \"c.h\"\n"
+                 "#include \"a.h\"\n"
+                 "// comment\n"
+                 "#include \"b.h\"\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"c.h\"\n"
+            "// comment\n"
+            "#include \"b.h\"\n",
+            sort("#include \"c.h\"\n"
+                 "#include \"a.h\"\n"
+                 "// comment\n"
+                 "#include \"b.h\"\n"));
+}
+
 TEST_F(SortIncludesTest, HandlesAngledIncludesAsSeparateBlocks) {
   EXPECT_EQ("#include \"a.h\"\n"
             "#include \"c.h\"\n"

                  "#include \"a.h\"\n"));
 }
 
+TEST_F(SortIncludesTest, RegroupsAngledIncludesInSeparateBlocks) {
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"c.h\"\n"
+            "\n"
+            "#include <b.h>\n"
+            "#include <d.h>\n",
+            sort("#include <d.h>\n"
+                 "#include <b.h>\n"
+                 "#include \"c.h\"\n"
+                 "#include \"a.h\"\n"));
+}
+
 TEST_F(SortIncludesTest, HandlesMultilineIncludes) {
   EXPECT_EQ("#include \"a.h\"\n"
             "#include \"b.h\"\n"

                  "a.cc"));
 }
 
+TEST_F(SortIncludesTest, RecognizeMainHeaderInAllGroups) {
+  Style.IncludeIsMainRegex = "([-_](test|unittest))?$";
+  Style.IncludeBlocks = FormatStyle::IBS_Merge;
+
+  EXPECT_EQ("#include \"c.h\"\n"
+            "#include \"a.h\"\n"
+            "#include \"b.h\"\n",
+            sort("#include \"b.h\"\n"
+                 "\n"
+                 "#include \"a.h\"\n"
+                 "#include \"c.h\"\n",
+                 "c.cc"));
+}
+
+TEST_F(SortIncludesTest, MainHeaderIsSeparatedWhenRegroupping) {
+  Style.IncludeIsMainRegex = "([-_](test|unittest))?$";
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+
+  EXPECT_EQ("#include \"a.h\"\n"
+            "\n"
+            "#include \"b.h\"\n"
+            "#include \"c.h\"\n",
+            sort("#include \"b.h\"\n"
+                 "\n"
+                 "#include \"a.h\"\n"
+                 "#include \"c.h\"\n",
+                 "a.cc"));
+}
+
 TEST_F(SortIncludesTest, SupportCaseInsensitiveMatching) {
   // Setup an regex for main includes so we can cover those as well.
   Style.IncludeIsMainRegex = "([-_](test|unittest))?$";

                  "c_main.cc"));
 }
 
+TEST_F(SortIncludesTest, PriorityGroupsAreSeparatedWhenRegroupping) {
+  Style.IncludeCategories = {{".*important_os_header.*", -1}, {".*", 1}};
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+
+  EXPECT_EQ("#include \"important_os_header.h\"\n"
+            "\n"
+            "#include \"c_main.h\"\n"
+            "\n"
+            "#include \"a_other.h\"\n",
+            sort("#include \"c_main.h\"\n"
+                 "#include \"a_other.h\"\n"
+                 "#include \"important_os_header.h\"\n",
+                 "c_main.cc"));
+
+  // check stable when re-run
+  EXPECT_EQ("#include \"important_os_header.h\"\n"
+            "\n"
+            "#include \"c_main.h\"\n"
+            "\n"
+            "#include \"a_other.h\"\n",
+            sort("#include \"important_os_header.h\"\n"
+                 "\n"
+                 "#include \"c_main.h\"\n"
+                 "\n"
+                 "#include \"a_other.h\"\n",
+                 "c_main.cc"));
+}
+
 TEST_F(SortIncludesTest, CalculatesCorrectCursorPosition) {
   std::string Code = "#include <ccc>\n"    // Start of line: 0
                      "#include <bbbbbb>\n" // Start of line: 15

                  "#include <b>\n"
                  "#include <b>\n"
                  "#include <c>\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Merge;
+  EXPECT_EQ("#include <a>\n"
+            "#include <b>\n"
+            "#include <c>\n",
+            sort("#include <a>\n"
+                 "#include <b>\n"
+                 "\n"
+                 "#include <b>\n"
+                 "\n"
+                 "#include <b>\n"
+                 "#include <c>\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+  EXPECT_EQ("#include <a>\n"
+            "#include <b>\n"
+            "#include <c>\n",
+            sort("#include <a>\n"
+                 "#include <b>\n"
+                 "\n"
+                 "#include <b>\n"
+                 "\n"
+                 "#include <b>\n"
+                 "#include <c>\n"));
 }
 
 TEST_F(SortIncludesTest, SortAndDeduplicateIncludes) {

                  "#include <b>\n"
                  "#include <c>\n"
                  "#include <b>\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Merge;
+  EXPECT_EQ("#include <a>\n"
+            "#include <b>\n"
+            "#include <c>\n",
+            sort("#include <b>\n"
+                 "#include <a>\n"
+                 "\n"
+                 "#include <b>\n"
+                 "\n"
+                 "#include <c>\n"
+                 "#include <b>\n"));
+
+  Style.IncludeBlocks = FormatStyle::IBS_Regroup;
+  EXPECT_EQ("#include <a>\n"
+            "#include <b>\n"
+            "#include <c>\n",
+            sort("#include <b>\n"
+                 "#include <a>\n"
+                 "\n"
+                 "#include <b>\n"
+                 "\n"
+                 "#include <c>\n"
+                 "#include <b>\n"));
 }
 
 TEST_F(SortIncludesTest, CalculatesCorrectCursorPositionAfterDeduplicate) {