[pseudo] Strip directives from a token stream

[sammccall]

Summary

This includes only the taken branch of conditional sections.
The API allows for producing a stream for a particular PP branch, which
will be used later for the secondary GLR parses of not-taken branches.

[hokein]

clang-tools-extra/pseudo/lib/DirectiveTree.cpp

373

return, otherwise we we hit the unreachable code below.

clang-tools-extra/pseudo/tool/ClangPseudo.cpp

40

preprocess seems a little confusing (the name reminds me of the traditional preprocessor process where comments are stripped, but not in our case here). may be strip-directives but it doesn't reflect the conditional-#if token work, but I think it is fine.

This flag is a feature-control flag for print-source and print-tokens, can we group three of them together?

[hokein]

would be nice to land it in some of form.

clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp

316

nit: cover #include as well?

[sammccall]

clang-tools-extra/pseudo/lib/DirectiveTree.cpp

373

That's deliberate, K_Empty is invalid.

clang-tools-extra/pseudo/tool/ClangPseudo.cpp

40

may be strip-directives but it doesn't reflect the conditional-#if token work, but I think it is fine

Agreed, renamed it.

This flag is a feature-control flag for print-source and print-tokens, can we group three of them together?

It's not really, it also affects the GLR parser etc.
Generally I think we should try to make the flags in this tool more orthogonal rather than clustering them by effects. Thematically it's more closely related to -print-directive-tree.

[sammccall]

rebase
add include to testcase
rename preprocess => strip-includes, and testcase
move print directivetree test to strip-includes.c
fix tests to not cook before parsing directivetree

clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h

struct DirectiveTree {
 
   /// Some piece of the file. {One of Code, Directive, Conditional}.
   class Chunk; // Defined below.
   std::vector<Chunk> Chunks;
 
   /// Extract preprocessor structure by examining the raw tokens.
   static DirectiveTree parse(const TokenStream &);
 
-  // FIXME: allow deriving a preprocessed stream
+  /// Produce a parseable token stream by stripping all directive tokens.
+  ///
+  /// Conditional sections are replaced by the taken branch, if any.
+  /// This tree must describe the provided token stream.
+  TokenStream stripDirectives(const TokenStream &) const;
 };
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree &);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree::Chunk &);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree::Code &);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                               const DirectiveTree::Directive &);
 llvm::raw_ostream &operator<<(llvm::raw_ostream &,
                               const DirectiveTree::Conditional &);

clang-tools-extra/pseudo/lib/DirectiveTree.cpp

 };
 
 } // namespace
 
 void chooseConditionalBranches(DirectiveTree &Tree, const TokenStream &Code) {
   BranchChooser{Code}.choose(Tree);
 }
 
+namespace {
+class Preprocessor {
+  const TokenStream &In;
+  TokenStream &Out;
+
+public:
+  Preprocessor(const TokenStream &In, TokenStream &Out) : In(In), Out(Out) {}
+  ~Preprocessor() { Out.finalize(); }
+
+  void walk(const DirectiveTree &T) {
+    for (const auto &C : T.Chunks)
+      walk(C);
+  }
+
+  void walk(const DirectiveTree::Chunk &C) {
+    switch (C.kind()) {
+    case DirectiveTree::Chunk::K_Code:
+      return walk((const DirectiveTree::Code &)C);
+    case DirectiveTree::Chunk::K_Directive:
+      return walk((const DirectiveTree::Directive &)C);
+    case DirectiveTree::Chunk::K_Conditional:
+      return walk((const DirectiveTree::Conditional &)C);
+    case DirectiveTree::Chunk::K_Empty:
+      break;
+    }
+    llvm_unreachable("bad chunk kind");
+  }
+
+  void walk(const DirectiveTree::Code &C) {
+    for (const auto &Tok : In.tokens(C.Tokens))
+      Out.push(Tok);
+  }
+
+  void walk(const DirectiveTree::Directive &) {}
+
+  void walk(const DirectiveTree::Conditional &C) {
+    if (C.Taken)
+      walk(C.Branches[*C.Taken].second);
+  }
+};
+} // namespace
+
+TokenStream DirectiveTree::stripDirectives(const TokenStream &In) const {
+  TokenStream Out;
+  Preprocessor(In, Out).walk(*this);
+  return Out;
+}
+
 } // namespace pseudo
 } // namespace clang

clang-tools-extra/pseudo/test/preprocess.c

+#include <stdio.h>
+int main() {
+#error This was inevitable...
+#if HELLO
+  printf("hello, world\n");
+  return 0;
+#else
+  abort();
+#endif
+}
+
+/* This comment gets lexed along with the input above! We just don't CHECK it.
+
+RUN: clang-pseudo -source %s -preprocess -print-source | FileCheck %s --strict-whitespace
+     CHECK: int main() {
+CHECK-NEXT:   printf("hello, world\n");
+CHECK-NEXT:   return 0;
+CHECK-NEXT: }
+
+*******************************************************************************/
+

clang-tools-extra/pseudo/tool/ClangPseudo.cpp

 #include "clang-pseudo/Token.h"
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 using clang::pseudo::Grammar;
+using clang::pseudo::TokenStream;
 using llvm::cl::desc;
 using llvm::cl::init;
 using llvm::cl::opt;
 
 static opt<std::string>
     Grammar("grammar", desc("Parse and check a BNF grammar file."), init(""));
 static opt<bool> PrintGrammar("print-grammar", desc("Print the grammar."));
 static opt<bool> PrintGraph("print-graph",
                             desc("Print the LR graph for the grammar"));
 static opt<bool> PrintTable("print-table",
                             desc("Print the LR table for the grammar"));
 static opt<std::string> Source("source", desc("Source file"));
 static opt<bool> PrintSource("print-source", desc("Print token stream"));
 static opt<bool> PrintTokens("print-tokens", desc("Print detailed token info"));
 static opt<bool>
     PrintDirectiveTree("print-directive-tree",
                       desc("Print directive structure of source code"));
+static opt<bool>
+    Preprocess("preprocess",
+               desc("Strip directives and select conditional sections"));
 
 static std::string readOrDie(llvm::StringRef Path) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
       llvm::MemoryBuffer::getFile(Path);
   if (std::error_code EC = Text.getError()) {
     llvm::errs() << "Error: can't read grammar file '" << Path
                  << "': " << EC.message() << "\n";
     ::exit(1);
if (Grammar.getNumOccurrences()) {
     if (PrintTable)
       llvm::outs() << clang::pseudo::LRTable::buildSLR(*G).dumpForTests(*G);
     return 0;
   }
 
   if (Source.getNumOccurrences()) {
     std::string Text = readOrDie(Source);
     clang::LangOptions LangOpts; // FIXME: use real options.
-    auto Stream = clang::pseudo::lex(Text, LangOpts);
-    auto Structure = clang::pseudo::DirectiveTree::parse(Stream);
-    clang::pseudo::chooseConditionalBranches(Structure, Stream);
+    TokenStream Raw = clang::pseudo::lex(Text, LangOpts);
+    TokenStream *Stream = &Raw;
+    auto Structure = clang::pseudo::DirectiveTree::parse(*Stream);
+    clang::pseudo::chooseConditionalBranches(Structure, *Stream);
 
     if (PrintDirectiveTree)
       llvm::outs() << Structure;
+    llvm::Optional<TokenStream> Preprocessed;
+    if (Preprocess) {
+      Preprocessed = Structure.stripDirectives(*Stream);
+      Stream = Preprocessed.getPointer();
+    }
+
     if (PrintSource)
-      Stream.print(llvm::outs());
+      Stream->print(llvm::outs());
     if (PrintTokens)
-      llvm::outs() << Stream;
+      llvm::outs() << *Stream;
   }
 
   return 0;
 }

clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp

 
 using testing::_;
 using testing::ElementsAre;
 using testing::Matcher;
 using testing::Pair;
 using testing::StrEq;
 using Chunk = DirectiveTree::Chunk;
 
-MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
+// Matches text of a list of tokens against a string (joined with spaces).
+// e.g. EXPECT_THAT(Stream.tokens(), tokens("int main ( ) { }"));
+MATCHER_P(tokens, Tokens, "") {
   std::vector<llvm::StringRef> Texts;
-  for (const Token &Tok : TS.tokens(arg.Tokens))
+  for (const Token &Tok : arg)
     Texts.push_back(Tok.text());
   return Matcher<std::string>(StrEq(Tokens))
       .MatchAndExplain(llvm::join(Texts, " "), result_listener);
 }
 
+// Matches tokens covered a directive chunk (with a Tokens property) against a
+// string, similar to tokens() above.
+// e.g. EXPECT_THAT(SomeDirective, tokensAre(Stream, "# include < vector >"));
+MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) {
+  return testing::Matches(tokens(Tokens))(TS.tokens(arg.Tokens));
+}
+
 MATCHER_P(chunkKind, K, "") { return arg.kind() == K; }
 
 TEST(DirectiveTree, Parse) {
   LangOptions Opts;
   std::string Code = R"cpp(
   #include <foo.h>
 
   int main() {
std::function<void(const DirectiveTree &)> Verify =
         };
 
     DirectiveTree Tree = DirectiveTree::parse(S);
     chooseConditionalBranches(Tree, S);
     Verify(Tree);
   }
 }
 
+TEST(DirectiveTree, StripDirectives) {
+  LangOptions Opts;
+  std::string Code = R"cpp(
+    a a a
+    #warning AAA
+    b b b
+    #if 1
+      c c c
+      #warning BBB
+      #if 0
+        d d d
+        #warning CC
+      #else
+        e e e
+      #endif
+      f f f
+      #if 0
+        g g g
+      #endif
+      h h h
+    #else
+      i i i
+    #endif
+    j j j
+  )cpp";
+  TokenStream S = cook(lex(Code, Opts), Opts);
+
+  DirectiveTree Tree = DirectiveTree::parse(S);
+  chooseConditionalBranches(Tree, S);
+  EXPECT_THAT(Tree.stripDirectives(S).tokens(),
+              tokens("a a a b b b c c c e e e f f f h h h j j j"));
+
+  const DirectiveTree &Part =
+      ((const DirectiveTree::Conditional &)Tree.Chunks[3]).Branches[0].second;
+  EXPECT_THAT(Part.stripDirectives(S).tokens(),
+              tokens("c c c e e e f f f h h h"));
+}
+
 } // namespace
 } // namespace pseudo
 } // namespace clang