add -dump-tokens-bg option for clang

[Anarion-zuo]

Summary

This path is intended it add -dump-tokens-bg option to clang.

While trying to extract information from code, we find it difficult to use -dump-tokens option with complex compiling systems, e.g. this one. This new option is supposed not to interfere with other parts of the build, and output token dumps to stdout.

A dump may look like this. Other formats are possible if one cares to propose.

file token dump begins hello.c
raw_identifier	'int'
raw_identifier	'foo'
l_paren	'('
r_paren	')'
l_brace	'{'
raw_identifier	'int'
raw_identifier	'i'
equal	'='
numeric_constant	'0'
semi	';'
raw_identifier	'int'
raw_identifier	'j'
equal	'='
raw_identifier	'i'
minus	'-'
numeric_constant	'4'
semi	';'
raw_identifier	'int'
raw_identifier	'k'
equal	'='
numeric_constant	'999'
slash	'/'
raw_identifier	'j'
semi	';'
raw_identifier	'return'
numeric_constant	'1'
semi	';'
r_brace	'}'
raw_identifier	'int'
raw_identifier	'main'
l_paren	'('
r_paren	')'
l_brace	'{'
raw_identifier	'foo'
l_paren	'('
r_paren	')'
semi	';'
raw_identifier	'return'
numeric_constant	'0'
semi	';'
r_brace	'}'
eof	''
file token dump ends hello.c

[Anarion-zuo]

It's my first time here. I don't know if I got everything right.

CodeOwners.rst says you guys are relevant somewhat. Thanks! @ChuanqiXu @jansvoboda11

[ChuanqiXu]

What is the intended use case of this?

[Anarion-zuo]

We intend to build a deep learning model with code as input, then noticed clang does not currently have a convenient way of dumping tokens in huge projects.

The dumped tokens are supposed to be converted to vectors, then given to a deep learning model.

[ChuanqiXu]

I think such requirement should be done by something like a plugin or an analysis tool by using clang as a library instead of implementing it in clang compiler itself directly.

[Anarion-zuo]

I see that plugin is a more elegant way of doing this. Do you still want this as a part of LLVM?

[ChuanqiXu]

In D152416#4404862, @Anarion-zuo wrote:

I see that plugin is a more elegant way of doing this. Do you still want this as a part of LLVM?

No, I don't feel this is needed.

clang/include/clang/Driver/Options.td

defm recovery_ast : BoolOption<"f", "recovery-ast",
   LangOpts<"RecoveryAST">, DefaultTrue,
   NegFlag<SetFalse>, PosFlag<SetTrue, [], "Preserve expressions in AST rather "
                               "than dropping them when encountering semantic errors">>;
 defm recovery_ast_type : BoolOption<"f", "recovery-ast-type",
   LangOpts<"RecoveryASTType">, DefaultTrue,
   NegFlag<SetFalse>, PosFlag<SetTrue, [], "Preserve the type for recovery "
                               "expressions when possible">>;
 
+def dump_tokens_bg : Flag<["-"], "dump-tokens-bg">,
+  HelpText<"Dump internal rep of tokens in the backgroud">;
+
 let Group = Action_Group in {
 
 def Eonly : Flag<["-"], "Eonly">,
   HelpText<"Just run preprocessor, no output (for timings)">;
 def dump_raw_tokens : Flag<["-"], "dump-raw-tokens">,
   HelpText<"Lex file in raw mode and dump raw tokens">;
 def analyze : Flag<["-"], "analyze">,
   HelpText<"Run static analysis engine">;

clang/include/clang/Frontend/FrontendOptions.h

public:
   /// Whether to share the FileManager when building modules.
   unsigned ModulesShareFileManager : 1;
 
   CodeCompleteOptions CodeCompleteOpts;
 
   /// Specifies the output format of the AST.
   ASTDumpOutputFormat ASTDumpFormat = ADOF_Default;
 
+  /// Whether to dump tokens to file in the background
+  unsigned DumpTokensBg : 1;
+
   enum {
     ARCMT_None,
     ARCMT_Check,
     ARCMT_Modify,
     ARCMT_Migrate
   } ARCMTAction = ARCMT_None;
 
   enum {

clang/lib/Frontend/CompilerInvocation.cpp

#undef FRONTEND_OPTION_WITH_MARSHALLING
   if (Opts.ProgramAction == frontend::FixIt && !Opts.FixItSuffix.empty()) {
     GenerateProgramAction = [&]() {
       GenerateArg(Args, OPT_fixit_EQ, Opts.FixItSuffix, SA);
     };
   }
 
   GenerateProgramAction();
 
+  if (Opts.DumpTokensBg == 1) {
+    GenerateArg(Args, OPT_dump_tokens_bg, SA);
+  }
+
   for (const auto &PluginArgs : Opts.PluginArgs) {
     Option Opt = getDriverOptTable().getOption(OPT_plugin_arg);
     const char *Spelling =
         SA(Opt.getPrefix() + Opt.getName() + PluginArgs.first);
     for (const auto &PluginArg : PluginArgs.second)
       denormalizeString(Args, Spelling, SA, Opt.getKind(), 0, PluginArg);
   }
 
if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) {
     if (Opts.CodeCompletionAt.FileName.empty())
       Diags.Report(diag::err_drv_invalid_value)
         << A->getAsString(Args) << A->getValue();
   }
 
   Opts.Plugins = Args.getAllArgValues(OPT_load);
   Opts.ASTDumpDecls = Args.hasArg(OPT_ast_dump, OPT_ast_dump_EQ);
   Opts.ASTDumpAll = Args.hasArg(OPT_ast_dump_all, OPT_ast_dump_all_EQ);
+  Opts.DumpTokensBg = Args.hasArg(OPT_dump_tokens_bg);
   // Only the -fmodule-file=<file> form.
   for (const auto *A : Args.filtered(OPT_fmodule_file)) {
     StringRef Val = A->getValue();
     if (!Val.contains('='))
       Opts.ModuleFiles.push_back(std::string(Val));
   }
 
   if (Opts.ProgramAction != frontend::GenerateModule && Opts.IsSystemModule)

clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp

#endif
   // If there were errors in processing arguments, don't do anything else.
   if (Clang->getDiagnostics().hasErrorOccurred())
     return false;
   // Create and execute the frontend action.
   std::unique_ptr<FrontendAction> Act(CreateFrontendAction(*Clang));
   if (!Act)
     return false;
   bool Success = Clang->ExecuteAction(*Act);
+  if (Clang->getFrontendOpts().DumpTokensBg) {
+    SourceManager &SM = Clang->getSourceManager();
+    FileID FID = SM.getMainFileID();
+    auto FileName = SM.getNonBuiltinFilenameForID(FID);
+    Lexer TheLexer(FID, SM.getBufferOrFake(FID), SM, Clang->getLangOpts());
+    auto &OS = llvm::outs();
+    OS << "file token dump begins " << FileName << '\n';
+    Token RawTok;
+    do {
+      TheLexer.LexFromRawLexer(RawTok);
+      OS << tok::getTokenName(RawTok.getKind());
+      if (!RawTok.isAnnotation()) {
+        std::string Spelling =
+            TheLexer.getSpelling(RawTok, SM, Clang->getLangOpts());
+        OS << "\t'" << Spelling << "'\n";
+      }
+    } while (RawTok.isNot(tok::eof));
+    OS << "file token dump ends " << FileName << '\n';
+  }
   if (Clang->getFrontendOpts().DisableFree)
     llvm::BuryPointer(std::move(Act));
   return Success;
 }
 
 } // namespace clang

clang/test/Preprocessor/dumptokens_bg.c

+// RUN: %clang_cc1 -dump-tokens-bg %s | FileCheck %s
+// CHECK: file token dump begins {{.*}}
+// CHECK: raw_identifier
+// CHECK: foo
+// CHECK: file token dump ends {{.*}}
+
+int foo() {
+  return 0;
+}