[clang-check] Add tokens-dump in clang-check.

[hokein]

Summary

It is useful for syntax-tree developement.

[kbobyrev]

Thanks!

clang/tools/clang-check/ClangCheck.cpp

 static cl::opt<bool> FixWhatYouCan(
     "fix-what-you-can",
     cl::desc(Options.getOptionHelpText(options::OPT_fix_what_you_can)),
     cl::cat(ClangCheckCategory));
 
 static cl::opt<bool> SyntaxTreeDump("syntax-tree-dump",
                                     cl::desc("dump the syntax tree"),
                                     cl::cat(ClangCheckCategory));
+static cl::opt<bool> TokensDump("tokens-dump",
+                                cl::desc("dump the preprocessed tokens"),
+                                cl::cat(ClangCheckCategory));
 
 namespace {
 
 // FIXME: Move FixItRewriteInPlace from lib/Rewrite/Frontend/FrontendActions.cpp
 // into a header file and reuse that.
 class FixItOptions : public clang::FixItOptions {
 public:
   FixItOptions() {
public:
   std::unique_ptr<clang::ASTConsumer>
   CreateASTConsumer(clang::CompilerInstance &CI, StringRef InFile) override {
     class Consumer : public clang::ASTConsumer {
     public:
       Consumer(clang::CompilerInstance &CI) : Collector(CI.getPreprocessor()) {}
 
       void HandleTranslationUnit(clang::ASTContext &AST) override {
         clang::syntax::TokenBuffer TB = std::move(Collector).consume();
+        if (TokensDump)
+          llvm::outs() << TB.dumpForTests();
         clang::syntax::Arena A(AST.getSourceManager(), AST.getLangOpts(), TB);
         llvm::outs() << clang::syntax::buildSyntaxTree(A, AST)->dump(
             AST.getSourceManager());
       }
 
     private:
       clang::syntax::TokenCollector Collector;
     };
int main(int argc, const char **argv) {
   ClangCheckActionFactory CheckFactory;
   std::unique_ptr<FrontendActionFactory> FrontendFactory;
 
   // Choose the correct factory based on the selected mode.
   if (Analyze)
     FrontendFactory = newFrontendActionFactory<clang::ento::AnalysisAction>();
   else if (Fixit)
     FrontendFactory = newFrontendActionFactory<ClangCheckFixItAction>();
-  else if (SyntaxTreeDump)
+  else if (SyntaxTreeDump || TokensDump)
     FrontendFactory = newFrontendActionFactory<DumpSyntaxTree>();
   else
     FrontendFactory = newFrontendActionFactory(&CheckFactory);
 
   return Tool.run(FrontendFactory.get());
 }