[clang-scan-deps] Add minimizer support for C++20 modules.

[Bigcheese]

Summary

This only adds support to the minimizer, it doesn't actually capture the dependencies yet.

[arphaman]

LGTM

[arphaman]

unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp

577

Looks like it's not indented according to clang-format rules

[Bigcheese]

Fixed and committed as r368381.

include/clang/Lex/DependencyDirectivesSourceMinimizer.h

enum TokenKind {
   pp_include_next,
   pp_if,
   pp_ifdef,
   pp_ifndef,
   pp_elif,
   pp_else,
   pp_endif,
   decl_at_import,
+  cxx_export_decl,
+  cxx_module_decl,
+  cxx_import_decl,
   pp_eof,
 };
 
 /// Represents a simplified token that's lexed as part of the source
 /// minimization. It's used to track the location of various preprocessor
 /// directives that could potentially have an effect on the depedencies.
 struct Token {
   /// The kind of token.

lib/Lex/DependencyDirectivesSourceMinimizer.cpp

private:
   ///
   /// \pre First points at a valid identifier head.
   LLVM_NODISCARD IdInfo lexIdentifier(const char *First, const char *const End);
   LLVM_NODISCARD bool isNextIdentifier(StringRef Id, const char *&First,
                                        const char *const End);
   LLVM_NODISCARD bool minimizeImpl(const char *First, const char *const End);
   LLVM_NODISCARD bool lexPPLine(const char *&First, const char *const End);
   LLVM_NODISCARD bool lexAt(const char *&First, const char *const End);
+  LLVM_NODISCARD bool lexModule(const char *&First, const char *const End);
   LLVM_NODISCARD bool lexDefine(const char *&First, const char *const End);
   LLVM_NODISCARD bool lexPragma(const char *&First, const char *const End);
   LLVM_NODISCARD bool lexEndif(const char *&First, const char *const End);
   LLVM_NODISCARD bool lexDefault(TokenKind Kind, StringRef Directive,
                                  const char *&First, const char *const End);
   Token &makeToken(TokenKind K) {
     Tokens.emplace_back(K, Out.size());
     return Tokens.back();
if (First == End)
     return false;
   if (!isVerticalWhitespace(*First))
     return reportError(
         ImportLoc, diag::err_dep_source_minimizer_unexpected_tokens_at_import);
   skipNewline(First, End);
   return false;
 }
 
+bool Minimizer::lexModule(const char *&First, const char *const End) {
+  IdInfo Id = lexIdentifier(First, End);
+  First = Id.Last;
+  bool Export = false;
+  if (Id.Name == "export") {
+    Export = true;
+    skipWhitespace(First, End);
+    if (!isIdentifierBody(*First)) {
+      skipLine(First, End);
+      return false;
+    }
+    Id = lexIdentifier(First, End);
+    First = Id.Last;
+  }
+
+  if (Id.Name != "module" && Id.Name != "import") {
+    skipLine(First, End);
+    return false;
+  }
+
+  skipWhitespace(First, End);
+
+  // Ignore this as a module directive if the next character can't be part of
+  // an import.
+
+  switch (*First) {
+  case ':':
+  case '<':
+  case '"':
+    break;
+  default:
+    if (!isIdentifierBody(*First)) {
+      skipLine(First, End);
+      return false;
+    }
+  }
+
+  if (Export) {
+    makeToken(cxx_export_decl);
+    append("export ");
+  }
+
+  if (Id.Name == "module")
+    makeToken(cxx_module_decl);
+  else
+    makeToken(cxx_import_decl);
+  append(Id.Name);
+  append(" ");
+  printToNewline(First, End);
+  append("\n");
+  return false;
+}
+
 bool Minimizer::lexDefine(const char *&First, const char *const End) {
   makeToken(pp_define);
   append("#define ");
   skipWhitespace(First, End);
 
   if (!isIdentifierHead(*First))
     return reportError(First, diag::err_pp_macro_not_identifier);
 

 bool Minimizer::lexDefault(TokenKind Kind, StringRef Directive,
                            const char *&First, const char *const End) {
   makeToken(Kind);
   put('#').append(Directive).put(' ');
   printDirectiveBody(First, End);
   return false;
 }
 
+static bool isStartOfRelevantLine(char First) {
+  switch (First) {
+  case '#':
+  case '@':
+  case 'i':
+  case 'e':
+  case 'm':
+    return true;
+  }
+  return false;
+}
+
 bool Minimizer::lexPPLine(const char *&First, const char *const End) {
   assert(First != End);
 
   skipWhitespace(First, End);
   assert(First <= End);
   if (First == End)
     return false;
 
-  if (*First != '#' && *First != '@') {
+  if (!isStartOfRelevantLine(*First)) {
     skipLine(First, End);
     assert(First <= End);
     return false;
   }
 
   // Handle "@import".
   if (*First == '@')
     return lexAt(First, End);
 
+  if (*First == 'i' || *First == 'e' || *First == 'm')
+    return lexModule(First, End);
+
   // Handle preprocessing directives.
   ++First; // Skip over '#'.
   skipWhitespace(First, End);
 
   if (First == End)
     return reportError(First, diag::err_pp_expected_eol);
 
   if (!isIdentifierHead(*First)) {

unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp

ASSERT_FALSE(
                                            "#ifndef A\n"
                                            "#elif A\n"
                                            "#else\n"
                                            "#include <A>\n"
                                            "#include_next <A>\n"
                                            "#__include_macros <A>\n"
                                            "#import <A>\n"
                                            "@import A;\n"
-                                           "#pragma clang module import A\n",
+                                           "#pragma clang module import A\n"
+                                           "export module m;\n"
+                                           "import m;\n",
                                            Out, Tokens));
   EXPECT_EQ(pp_define, Tokens[0].K);
   EXPECT_EQ(pp_undef, Tokens[1].K);
   EXPECT_EQ(pp_endif, Tokens[2].K);
   EXPECT_EQ(pp_if, Tokens[3].K);
   EXPECT_EQ(pp_ifdef, Tokens[4].K);
   EXPECT_EQ(pp_ifndef, Tokens[5].K);
   EXPECT_EQ(pp_elif, Tokens[6].K);
   EXPECT_EQ(pp_else, Tokens[7].K);
   EXPECT_EQ(pp_include, Tokens[8].K);
   EXPECT_EQ(pp_include_next, Tokens[9].K);
   EXPECT_EQ(pp___include_macros, Tokens[10].K);
   EXPECT_EQ(pp_import, Tokens[11].K);
   EXPECT_EQ(decl_at_import, Tokens[12].K);
   EXPECT_EQ(pp_pragma_import, Tokens[13].K);
-  EXPECT_EQ(pp_eof, Tokens[14].K);
+  EXPECT_EQ(cxx_export_decl, Tokens[14].K);
+  EXPECT_EQ(cxx_module_decl, Tokens[15].K);
+  EXPECT_EQ(cxx_import_decl, Tokens[16].K);
+  EXPECT_EQ(pp_eof, Tokens[17].K);
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest, Define) {
   SmallVector<char, 128> Out;
   SmallVector<Token, 4> Tokens;
 
   ASSERT_FALSE(
       minimizeSourceToDependencyDirectives("#define MACRO", Out, Tokens));
Source = R"(// comment
     #pragma once extra tokens
     // another comment
     #include <test.h>
     )";
   ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out));
   EXPECT_STREQ("#pragma once\n#include <test.h>\n", Out.data());
 }
 
+TEST(MinimizeSourceToDependencyDirectivesTest, CxxModules) {
+    SmallVector<char, 128> Out;
+    SmallVector<Token, 4> Tokens;
+
+    StringRef Source = R"(
+      module;
+      #include "textual-header.h"
+
+      export module m;
+      exp\
+ort \
+        import \
+        :l [[rename]];
+
+      export void f();
+
+      void h() {
+        import.a = 3;
+        import = 3;
+        import <<= 3;
+        import->a = 3;
+        import();
+        import . a();
+
+        import a b d e d e f e;
+        import foo [[no_unique_address]];
+        import foo();
+        import f(:sefse);
+        import f(->a = 3);
+      }
+      )";
+    ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Tokens));
+    EXPECT_STREQ("#include \"textual-header.h\"\nexport module m;\n"
+                 "export import :l [[rename]];\n"
+                 "import <<= 3;\nimport a b d e d e f e;\n"
+                 "import foo [[no_unique_address]];\nimport foo();\n"
+                 "import f(:sefse);\nimport f(->a = 3);\n", Out.data());
+    ASSERT_EQ(Tokens.size(), 12u);
+    EXPECT_EQ(Tokens[0].K,
+              minimize_source_to_dependency_directives::pp_include);
+    EXPECT_EQ(Tokens[2].K,
+              minimize_source_to_dependency_directives::cxx_module_decl);
+}
+
 } // end anonymous namespace