diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -6119,6 +6119,9 @@ NegFlag, PosFlag>; +def dump_tokens_bg : Flag<["-"], "dump-tokens-bg">, + HelpText<"Dump internal rep of tokens in the backgroud">; + let Group = Action_Group in { def Eonly : Flag<["-"], "Eonly">, diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -352,6 +352,9 @@ /// Specifies the output format of the AST. ASTDumpOutputFormat ASTDumpFormat = ADOF_Default; + /// Whether to dump tokens to file in the background + unsigned DumpTokensBg : 1; + enum { ARCMT_None, ARCMT_Check, diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -2570,6 +2570,10 @@ GenerateProgramAction(); + if (Opts.DumpTokensBg == 1) { + GenerateArg(Args, OPT_dump_tokens_bg, SA); + } + for (const auto &PluginArgs : Opts.PluginArgs) { Option Opt = getDriverOptTable().getOption(OPT_plugin_arg); const char *Spelling = @@ -2780,6 +2784,7 @@ Opts.Plugins = Args.getAllArgValues(OPT_load); Opts.ASTDumpDecls = Args.hasArg(OPT_ast_dump, OPT_ast_dump_EQ); Opts.ASTDumpAll = Args.hasArg(OPT_ast_dump_all, OPT_ast_dump_all_EQ); + Opts.DumpTokensBg = Args.hasArg(OPT_dump_tokens_bg); // Only the -fmodule-file= form. for (const auto *A : Args.filtered(OPT_fmodule_file)) { StringRef Val = A->getValue(); diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp --- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp +++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp @@ -262,6 +262,25 @@ if (!Act) return false; bool Success = Clang->ExecuteAction(*Act); + if (Clang->getFrontendOpts().DumpTokensBg) { + SourceManager &SM = Clang->getSourceManager(); + FileID FID = SM.getMainFileID(); + auto FileName = SM.getNonBuiltinFilenameForID(FID); + Lexer TheLexer(FID, SM.getBufferOrFake(FID), SM, Clang->getLangOpts()); + auto &OS = llvm::outs(); + OS << "file token dump begins " << FileName << '\n'; + Token RawTok; + do { + TheLexer.LexFromRawLexer(RawTok); + OS << tok::getTokenName(RawTok.getKind()); + if (!RawTok.isAnnotation()) { + std::string Spelling = + TheLexer.getSpelling(RawTok, SM, Clang->getLangOpts()); + OS << "\t'" << Spelling << "'\n"; + } + } while (RawTok.isNot(tok::eof)); + OS << "file token dump ends " << FileName << '\n'; + } if (Clang->getFrontendOpts().DisableFree) llvm::BuryPointer(std::move(Act)); return Success; diff --git a/clang/test/Preprocessor/dumptokens_bg.c b/clang/test/Preprocessor/dumptokens_bg.c new file mode 100644 --- /dev/null +++ b/clang/test/Preprocessor/dumptokens_bg.c @@ -0,0 +1,9 @@ +// RUN: %clang_cc1 -dump-tokens-bg %s | FileCheck %s +// CHECK: file token dump begins {{.*}} +// CHECK: raw_identifier +// CHECK: foo +// CHECK: file token dump ends {{.*}} + +int foo() { + return 0; +}