[clang-format] Add option for case sensitive regexes for sorted includes

[HazardyKnusperkeks]

Summary

I think the title says everything.

[HazardyKnusperkeks]

Do I have to do something? For me it doesn't look like the build failure is due to my changes.
Locally I've only build and run the FormatTests target.

[MyDeveloperDay]

Can you rerun

clang/docs/tools/dump_style.py

[MyDeveloperDay]

make sure you build the other clang-extra tools (I'm not sure who else uses the lib/Tooling)

[HazardyKnusperkeks]

I've rebased it to master, ran the documentation script and have build the targets
clang-format clang-tidy pp-trace clang-include-fixer clangd clang-doc clang-move modularize clang-rename FormatTests successfully.

[MyDeveloperDay]

This LGTM

[HazardyKnusperkeks]

Thanks,

than someone needs to commit it.

Regards Björn.

clang/docs/ClangFormatStyleOptions.rst

**IncludeCategories** (``std::vector<IncludeCategory>``)
 
   There is a third and optional field ``SortPriority`` which can used while
   ``IncludeBloks = IBS_Regroup`` to define the priority in which ``#includes``
   should be ordered, and value of ``Priority`` defines the order of
   ``#include blocks`` and also enables to group ``#includes`` of different
   priority for order.``SortPriority`` is set to the value of ``Priority``
   as default if it is not assigned.
 
+  Each regular expression can be marked as case sensitive with the field
+  ``CaseSensitive``, per default it is not.
+
   To configure this in the .clang-format file, use:
 
   .. code-block:: yaml
 
     IncludeCategories:
       - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
         Priority:        2
         SortPriority:    2
+        CaseSensitive:   true
       - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
         Priority:        3
       - Regex:           '<[[:alnum:].]+>'
         Priority:        4
       - Regex:           '.*'
         Priority:        1
         SortPriority:    0
 

clang/include/clang/Tooling/Inclusions/IncludeStyle.h

struct IncludeStyle {
   /// See documentation of ``IncludeCategories``.
   struct IncludeCategory {
     /// The regular expression that this category matches.
     std::string Regex;
     /// The priority to assign to this category.
     int Priority;
     /// The custom priority to sort before grouping.
     int SortPriority;
+    /// If the regular expression is case sensitive.
+    bool RegexIsCaseSensitive;
     bool operator==(const IncludeCategory &Other) const {
-      return Regex == Other.Regex && Priority == Other.Priority;
+      return Regex == Other.Regex && Priority == Other.Priority &&
+             RegexIsCaseSensitive == Other.RegexIsCaseSensitive;
     }
   };
 
   /// Regular expressions denoting the different ``#include`` categories
   /// used for ordering ``#includes``.
   ///
   /// `POSIX extended
   /// <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html>`_
   /// regular expressions are supported.
   ///
   /// These regular expressions are matched against the filename of an include
   /// (including the <> or "") in order. The value belonging to the first
   /// matching regular expression is assigned and ``#includes`` are sorted first
   /// according to increasing category number and then alphabetically within
   /// each category.
   ///
   /// If none of the regular expressions match, INT_MAX is assigned as
   /// category. The main header for a source file automatically gets category 0.
   /// so that it is generally kept at the beginning of the ``#includes``
   /// (https://llvm.org/docs/CodingStandards.html#include-style). However, you
   /// can also assign negative priorities if you have certain headers that
   /// always need to be first.
   /// 
   /// There is a third and optional field ``SortPriority`` which can used while
   /// ``IncludeBloks = IBS_Regroup`` to define the priority in which ``#includes``
   /// should be ordered, and value of ``Priority`` defines the order of
   /// ``#include blocks`` and also enables to group ``#includes`` of different
   /// priority for order.``SortPriority`` is set to the value of ``Priority``
   /// as default if it is not assigned.
   ///
+  /// Each regular expression can be marked as case sensitive with the field
+  /// ``CaseSensitive``, per default it is not.
+  ///
   /// To configure this in the .clang-format file, use:
   /// \code{.yaml}
   ///   IncludeCategories:
   ///     - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
   ///       Priority:        2
   ///       SortPriority:    2
+  ///       CaseSensitive:   true
   ///     - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
   ///       Priority:        3
   ///     - Regex:           '<[[:alnum:].]+>'
   ///       Priority:        4
   ///     - Regex:           '.*'
   ///       Priority:        1
   ///       SortPriority:    0
   /// \endcode

clang/lib/Format/Format.cpp

FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.DeriveLineEnding = true;
   LLVMStyle.DerivePointerAlignment = false;
   LLVMStyle.ExperimentalAutoDetectBinPacking = false;
   LLVMStyle.FixNamespaceComments = true;
   LLVMStyle.ForEachMacros.push_back("foreach");
   LLVMStyle.ForEachMacros.push_back("Q_FOREACH");
   LLVMStyle.ForEachMacros.push_back("BOOST_FOREACH");
   LLVMStyle.IncludeStyle.IncludeCategories = {
-      {"^\"(llvm|llvm-c|clang|clang-c)/", 2, 0},
-      {"^(<|\"(gtest|gmock|isl|json)/)", 3, 0},
-      {".*", 1, 0}};
+      {"^\"(llvm|llvm-c|clang|clang-c)/", 2, 0, false},
+      {"^(<|\"(gtest|gmock|isl|json)/)", 3, 0, false},
+      {".*", 1, 0, false}};
   LLVMStyle.IncludeStyle.IncludeIsMainRegex = "(Test)?$";
   LLVMStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Preserve;
   LLVMStyle.IndentCaseLabels = false;
   LLVMStyle.IndentCaseBlocks = false;
   LLVMStyle.IndentGotoLabels = true;
   LLVMStyle.IndentPPDirectives = FormatStyle::PPDIS_None;
   LLVMStyle.IndentWrappedFunctionNames = false;
   LLVMStyle.IndentWidth = 2;
FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
   GoogleStyle.AlignEscapedNewlines = FormatStyle::ENAS_Left;
   GoogleStyle.AllowShortIfStatementsOnASingleLine =
       FormatStyle::SIS_WithoutElse;
   GoogleStyle.AllowShortLoopsOnASingleLine = true;
   GoogleStyle.AlwaysBreakBeforeMultilineStrings = true;
   GoogleStyle.AlwaysBreakTemplateDeclarations = FormatStyle::BTDS_Yes;
   GoogleStyle.ConstructorInitializerAllOnOneLineOrOnePerLine = true;
   GoogleStyle.DerivePointerAlignment = true;
-  GoogleStyle.IncludeStyle.IncludeCategories = {{"^<ext/.*\\.h>", 2, 0},
-                                                {"^<.*\\.h>", 1, 0},
-                                                {"^<.*", 2, 0},
-                                                {".*", 3, 0}};
+  GoogleStyle.IncludeStyle.IncludeCategories = {{"^<ext/.*\\.h>", 2, 0, false},
+                                                {"^<.*\\.h>", 1, 0, false},
+                                                {"^<.*", 2, 0, false},
+                                                {".*", 3, 0, false}};
   GoogleStyle.IncludeStyle.IncludeIsMainRegex = "([-_](test|unittest))?$";
   GoogleStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
   GoogleStyle.IndentCaseLabels = true;
   GoogleStyle.KeepEmptyLinesAtTheStartOfBlocks = false;
   GoogleStyle.ObjCBinPackProtocolList = FormatStyle::BPS_Never;
   GoogleStyle.ObjCSpaceAfterProperty = false;
   GoogleStyle.ObjCSpaceBeforeProtocolList = true;
   GoogleStyle.PointerAlignment = FormatStyle::PAS_Left;

clang/lib/Tooling/Inclusions/HeaderIncludes.cpp

StringRef matchingStem(llvm::StringRef Path) {
   return Name.substr(0, Name.find('.', 1));
 }
 
 } // anonymous namespace
 
 IncludeCategoryManager::IncludeCategoryManager(const IncludeStyle &Style,
                                                StringRef FileName)
     : Style(Style), FileName(FileName) {
-  for (const auto &Category : Style.IncludeCategories)
-    CategoryRegexs.emplace_back(Category.Regex, llvm::Regex::IgnoreCase);
+  for (const auto &Category : Style.IncludeCategories) {
+    CategoryRegexs.emplace_back(Category.Regex, Category.RegexIsCaseSensitive
+                                                    ? llvm::Regex::NoFlags
+                                                    : llvm::Regex::IgnoreCase);
+  }
   IsMainFile = FileName.endswith(".c") || FileName.endswith(".cc") ||
                FileName.endswith(".cpp") || FileName.endswith(".c++") ||
                FileName.endswith(".cxx") || FileName.endswith(".m") ||
                FileName.endswith(".mm");
   if (!Style.IncludeIsMainSourceRegex.empty()) {
     llvm::Regex MainFileRegex(Style.IncludeIsMainSourceRegex);
     IsMainFile |= MainFileRegex.match(FileName);
   }

clang/lib/Tooling/Inclusions/IncludeStyle.cpp

 namespace llvm {
 namespace yaml {
 
 void MappingTraits<IncludeStyle::IncludeCategory>::mapping(
     IO &IO, IncludeStyle::IncludeCategory &Category) {
   IO.mapOptional("Regex", Category.Regex);
   IO.mapOptional("Priority", Category.Priority);
   IO.mapOptional("SortPriority", Category.SortPriority);
+  IO.mapOptional("CaseSensitive", Category.RegexIsCaseSensitive);
 }
 
 void ScalarEnumerationTraits<IncludeStyle::IncludeBlocksStyle>::enumeration(
     IO &IO, IncludeStyle::IncludeBlocksStyle &Value) {
   IO.enumCase(Value, "Preserve", IncludeStyle::IBS_Preserve);
   IO.enumCase(Value, "Merge", IncludeStyle::IBS_Merge);
   IO.enumCase(Value, "Regroup", IncludeStyle::IBS_Regroup);
 }
 
 } // namespace yaml
 } // namespace llvm

clang/unittests/Format/FormatTest.cpp

TEST_F(FormatTest, ParsesConfiguration) {
   CHECK_PARSE("WhitespaceSensitiveMacros: ['STRINGIZE']",
               WhitespaceSensitiveMacros, std::vector<std::string>{"STRINGIZE"});
   CHECK_PARSE("WhitespaceSensitiveMacros: ['STRINGIZE', 'ASSERT']",
               WhitespaceSensitiveMacros,
               std::vector<std::string>({"STRINGIZE", "ASSERT"}));
 
   Style.IncludeStyle.IncludeCategories.clear();
   std::vector<tooling::IncludeStyle::IncludeCategory> ExpectedCategories = {
-      {"abc/.*", 2, 0}, {".*", 1, 0}};
+      {"abc/.*", 2, 0, false}, {".*", 1, 0, true}};
   CHECK_PARSE("IncludeCategories:\n"
               "  - Regex: abc/.*\n"
               "    Priority: 2\n"
               "  - Regex: .*\n"
-              "    Priority: 1",
+              "    Priority: 1\n"
+              "    CaseSensitive: true\n",
               IncludeStyle.IncludeCategories, ExpectedCategories);
   CHECK_PARSE("IncludeIsMainRegex: 'abc$'", IncludeStyle.IncludeIsMainRegex,
               "abc$");
   CHECK_PARSE("IncludeIsMainSourceRegex: 'abc$'",
               IncludeStyle.IncludeIsMainSourceRegex, "abc$");
 
   Style.RawStringFormats.clear();
   std::vector<FormatStyle::RawStringFormat> ExpectedRawStringFormats = {

clang/unittests/Format/SortIncludesTest.cpp

EXPECT_EQ("// comment\n"
                  "#include <b>\n"
                  "#include <a>\n",
                  {tooling::Range(25, 1)}));
 }
 
 TEST_F(SortIncludesTest, SortedIncludesUsingSortPriorityAttribute) {
   FmtStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
   FmtStyle.IncludeStyle.IncludeCategories = {
-      {"^<sys/param\\.h>", 1, 0},
-      {"^<sys/types\\.h>", 1, 1},
-      {"^<sys.*/", 1, 2},
-      {"^<uvm/", 2, 3},
-      {"^<machine/", 3, 4},
-      {"^<dev/", 4, 5},
-      {"^<net.*/", 5, 6},
-      {"^<protocols/", 5, 7},
-      {"^<(fs|miscfs|msdosfs|nfs|ntfs|ufs)/", 6, 8},
-      {"^<(x86|amd64|i386|xen)/", 7, 8},
-      {"<path", 9, 11},
-      {"^<[^/].*\\.h>", 8, 10},
-      {"^\".*\\.h\"", 10, 12}};
+      {"^<sys/param\\.h>", 1, 0, false},
+      {"^<sys/types\\.h>", 1, 1, false},
+      {"^<sys.*/", 1, 2, false},
+      {"^<uvm/", 2, 3, false},
+      {"^<machine/", 3, 4, false},
+      {"^<dev/", 4, 5, false},
+      {"^<net.*/", 5, 6, false},
+      {"^<protocols/", 5, 7, false},
+      {"^<(fs|miscfs|msdosfs|nfs|ntfs|ufs)/", 6, 8, false},
+      {"^<(x86|amd64|i386|xen)/", 7, 8, false},
+      {"<path", 9, 11, false},
+      {"^<[^/].*\\.h>", 8, 10, false},
+      {"^\".*\\.h\"", 10, 12, false}};
   EXPECT_EQ("#include <sys/param.h>\n"
             "#include <sys/types.h>\n"
             "#include <sys/ioctl.h>\n"
             "#include <sys/socket.h>\n"
             "#include <sys/stat.h>\n"
             "#include <sys/wait.h>\n"
             "\n"
             "#include <net/if.h>\n"
EXPECT_EQ("#include \"llvm/A.h\"\n"
                  "#include \"GTest/GTest.h\"\n"
                  "#include \"llvm/A.h\"\n"
                  "#include \"gmock/gmock.h\"\n"
                  "#include \"llvm/X.h\"\n"
                  "#include \"LLVM/z.h\"\n",
                  "a_TEST.cc"));
 }
 
+TEST_F(SortIncludesTest, SupportOptionalCaseSensitiveMachting) {
+  Style.IncludeBlocks = clang::tooling::IncludeStyle::IBS_Regroup;
+  Style.IncludeCategories = {{"^\"", 1, 0, false},
+                             {"^<.*\\.h>$", 2, 0, false},
+                             {"^<Q[A-Z][^\\.]*>", 3, 0, false},
+                             {"^<Qt[^\\.]*>", 4, 0, false},
+                             {"^<", 5, 0, false}};
+
+  StringRef UnsortedCode = "#include <QWidget>\n"
+                           "#include \"qt.h\"\n"
+                           "#include <algorithm>\n"
+                           "#include <windows.h>\n"
+                           "#include <QLabel>\n"
+                           "#include \"qa.h\"\n"
+                           "#include <queue>\n"
+                           "#include <qtwhatever.h>\n"
+                           "#include <QtGlobal>\n";
+
+  EXPECT_EQ("#include \"qa.h\"\n"
+            "#include \"qt.h\"\n"
+            "\n"
+            "#include <qtwhatever.h>\n"
+            "#include <windows.h>\n"
+            "\n"
+            "#include <QLabel>\n"
+            "#include <QWidget>\n"
+            "#include <QtGlobal>\n"
+            "#include <queue>\n"
+            "\n"
+            "#include <algorithm>\n",
+            sort(UnsortedCode));
+
+  Style.IncludeCategories[2].RegexIsCaseSensitive = true;
+  Style.IncludeCategories[3].RegexIsCaseSensitive = true;
+  EXPECT_EQ("#include \"qa.h\"\n"
+            "#include \"qt.h\"\n"
+            "\n"
+            "#include <qtwhatever.h>\n"
+            "#include <windows.h>\n"
+            "\n"
+            "#include <QLabel>\n"
+            "#include <QWidget>\n"
+            "\n"
+            "#include <QtGlobal>\n"
+            "\n"
+            "#include <algorithm>\n"
+            "#include <queue>\n",
+            sort(UnsortedCode));
+}
+
 TEST_F(SortIncludesTest, NegativePriorities) {
-  Style.IncludeCategories = {{".*important_os_header.*", -1, 0}, {".*", 1, 0}};
+  Style.IncludeCategories = {{".*important_os_header.*", -1, 0, false},
+                             {".*", 1, 0, false}};
   EXPECT_EQ("#include \"important_os_header.h\"\n"
             "#include \"c_main.h\"\n"
             "#include \"a_other.h\"\n",
             sort("#include \"c_main.h\"\n"
                  "#include \"a_other.h\"\n"
                  "#include \"important_os_header.h\"\n",
                  "c_main.cc"));
 
   // check stable when re-run
   EXPECT_EQ("#include \"important_os_header.h\"\n"
             "#include \"c_main.h\"\n"
             "#include \"a_other.h\"\n",
             sort("#include \"important_os_header.h\"\n"
                  "#include \"c_main.h\"\n"
                  "#include \"a_other.h\"\n",
                  "c_main.cc", 0));
 }
 
 TEST_F(SortIncludesTest, PriorityGroupsAreSeparatedWhenRegroupping) {
-  Style.IncludeCategories = {{".*important_os_header.*", -1, 0}, {".*", 1, 0}};
+  Style.IncludeCategories = {{".*important_os_header.*", -1, 0, false},
+                             {".*", 1, 0, false}};
   Style.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
 
   EXPECT_EQ("#include \"important_os_header.h\"\n"
             "\n"
             "#include \"c_main.h\"\n"
             "\n"
             "#include \"a_other.h\"\n",
             sort("#include \"c_main.h\"\n"