diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h b/clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h --- a/clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/DirectiveTree.h @@ -92,7 +92,11 @@ /// Extract preprocessor structure by examining the raw tokens. static DirectiveTree parse(const TokenStream &); - // FIXME: allow deriving a preprocessed stream + /// Produce a parseable token stream by stripping all directive tokens. + /// + /// Conditional sections are replaced by the taken branch, if any. + /// This tree must describe the provided token stream. + TokenStream stripDirectives(const TokenStream &) const; }; llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree &); llvm::raw_ostream &operator<<(llvm::raw_ostream &, const DirectiveTree::Chunk &); diff --git a/clang-tools-extra/pseudo/lib/DirectiveTree.cpp b/clang-tools-extra/pseudo/lib/DirectiveTree.cpp --- a/clang-tools-extra/pseudo/lib/DirectiveTree.cpp +++ b/clang-tools-extra/pseudo/lib/DirectiveTree.cpp @@ -347,5 +347,53 @@ BranchChooser{Code}.choose(Tree); } +namespace { +class Preprocessor { + const TokenStream &In; + TokenStream &Out; + +public: + Preprocessor(const TokenStream &In, TokenStream &Out) : In(In), Out(Out) {} + ~Preprocessor() { Out.finalize(); } + + void walk(const DirectiveTree &T) { + for (const auto &C : T.Chunks) + walk(C); + } + + void walk(const DirectiveTree::Chunk &C) { + switch (C.kind()) { + case DirectiveTree::Chunk::K_Code: + return walk((const DirectiveTree::Code &)C); + case DirectiveTree::Chunk::K_Directive: + return walk((const DirectiveTree::Directive &)C); + case DirectiveTree::Chunk::K_Conditional: + return walk((const DirectiveTree::Conditional &)C); + case DirectiveTree::Chunk::K_Empty: + break; + } + llvm_unreachable("bad chunk kind"); + } + + void walk(const DirectiveTree::Code &C) { + for (const auto &Tok : In.tokens(C.Tokens)) + Out.push(Tok); + } + + void walk(const DirectiveTree::Directive &) {} + + void walk(const DirectiveTree::Conditional &C) { + if (C.Taken) + walk(C.Branches[*C.Taken].second); + } +}; +} // namespace + +TokenStream DirectiveTree::stripDirectives(const TokenStream &In) const { + TokenStream Out; + Preprocessor(In, Out).walk(*this); + return Out; +} + } // namespace pseudo } // namespace clang diff --git a/clang-tools-extra/pseudo/test/preprocess.c b/clang-tools-extra/pseudo/test/preprocess.c new file mode 100644 --- /dev/null +++ b/clang-tools-extra/pseudo/test/preprocess.c @@ -0,0 +1,21 @@ +#include +int main() { +#error This was inevitable... +#if HELLO + printf("hello, world\n"); + return 0; +#else + abort(); +#endif +} + +/* This comment gets lexed along with the input above! We just don't CHECK it. + +RUN: clang-pseudo -source %s -preprocess -print-source | FileCheck %s --strict-whitespace + CHECK: int main() { +CHECK-NEXT: printf("hello, world\n"); +CHECK-NEXT: return 0; +CHECK-NEXT: } + +*******************************************************************************/ + diff --git a/clang-tools-extra/pseudo/tool/ClangPseudo.cpp b/clang-tools-extra/pseudo/tool/ClangPseudo.cpp --- a/clang-tools-extra/pseudo/tool/ClangPseudo.cpp +++ b/clang-tools-extra/pseudo/tool/ClangPseudo.cpp @@ -18,6 +18,7 @@ #include "llvm/Support/MemoryBuffer.h" using clang::pseudo::Grammar; +using clang::pseudo::TokenStream; using llvm::cl::desc; using llvm::cl::init; using llvm::cl::opt; @@ -35,6 +36,9 @@ static opt PrintDirectiveTree("print-directive-tree", desc("Print directive structure of source code")); +static opt + Preprocess("preprocess", + desc("Strip directives and select conditional sections")); static std::string readOrDie(llvm::StringRef Path) { llvm::ErrorOr> Text = @@ -73,16 +77,23 @@ if (Source.getNumOccurrences()) { std::string Text = readOrDie(Source); clang::LangOptions LangOpts; // FIXME: use real options. - auto Stream = clang::pseudo::lex(Text, LangOpts); - auto Structure = clang::pseudo::DirectiveTree::parse(Stream); - clang::pseudo::chooseConditionalBranches(Structure, Stream); + TokenStream Raw = clang::pseudo::lex(Text, LangOpts); + TokenStream *Stream = &Raw; + auto Structure = clang::pseudo::DirectiveTree::parse(*Stream); + clang::pseudo::chooseConditionalBranches(Structure, *Stream); if (PrintDirectiveTree) llvm::outs() << Structure; + llvm::Optional Preprocessed; + if (Preprocess) { + Preprocessed = Structure.stripDirectives(*Stream); + Stream = Preprocessed.getPointer(); + } + if (PrintSource) - Stream.print(llvm::outs()); + Stream->print(llvm::outs()); if (PrintTokens) - llvm::outs() << Stream; + llvm::outs() << *Stream; } return 0; diff --git a/clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp b/clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp --- a/clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp +++ b/clang-tools-extra/pseudo/unittests/DirectiveTreeTest.cpp @@ -27,14 +27,23 @@ using testing::StrEq; using Chunk = DirectiveTree::Chunk; -MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) { +// Matches text of a list of tokens against a string (joined with spaces). +// e.g. EXPECT_THAT(Stream.tokens(), tokens("int main ( ) { }")); +MATCHER_P(tokens, Tokens, "") { std::vector Texts; - for (const Token &Tok : TS.tokens(arg.Tokens)) + for (const Token &Tok : arg) Texts.push_back(Tok.text()); return Matcher(StrEq(Tokens)) .MatchAndExplain(llvm::join(Texts, " "), result_listener); } +// Matches tokens covered a directive chunk (with a Tokens property) against a +// string, similar to tokens() above. +// e.g. EXPECT_THAT(SomeDirective, tokensAre(Stream, "# include < vector >")); +MATCHER_P2(tokensAre, TS, Tokens, "tokens are " + std::string(Tokens)) { + return testing::Matches(tokens(Tokens))(TS.tokens(arg.Tokens)); +} + MATCHER_P(chunkKind, K, "") { return arg.kind() == K; } TEST(DirectiveTree, Parse) { @@ -301,6 +310,44 @@ } } +TEST(DirectiveTree, StripDirectives) { + LangOptions Opts; + std::string Code = R"cpp( + a a a + #warning AAA + b b b + #if 1 + c c c + #warning BBB + #if 0 + d d d + #warning CC + #else + e e e + #endif + f f f + #if 0 + g g g + #endif + h h h + #else + i i i + #endif + j j j + )cpp"; + TokenStream S = cook(lex(Code, Opts), Opts); + + DirectiveTree Tree = DirectiveTree::parse(S); + chooseConditionalBranches(Tree, S); + EXPECT_THAT(Tree.stripDirectives(S).tokens(), + tokens("a a a b b b c c c e e e f f f h h h j j j")); + + const DirectiveTree &Part = + ((const DirectiveTree::Conditional &)Tree.Chunks[3]).Branches[0].second; + EXPECT_THAT(Part.stripDirectives(S).tokens(), + tokens("c c c e e e f f f h h h")); +} + } // namespace } // namespace pseudo } // namespace clang