diff --git a/clang/lib/Format/CMakeLists.txt b/clang/lib/Format/CMakeLists.txt --- a/clang/lib/Format/CMakeLists.txt +++ b/clang/lib/Format/CMakeLists.txt @@ -8,6 +8,7 @@ FormatToken.cpp FormatTokenLexer.cpp MacroExpander.cpp + MacroUnexpander.cpp NamespaceEndCommentsFixer.cpp SortJavaScriptImports.cpp TokenAnalyzer.cpp diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -436,6 +436,15 @@ // in a configured macro expansion. llvm::Optional MacroCtx; + /// When macro expansion introduces nodes with children, those are marked as + /// \c MacroParent. + /// FIXME: The formatting code currently hard-codes the assumption that + /// child nodes are introduced by blocks following an opening brace. + /// This is deeply baked into the code and disentangling this will require + /// signficant refactorings. \c MacroParent allows us to special-case the + /// cases in which we treat parents as block-openers for now. + bool MacroParent = false; + bool is(tok::TokenKind Kind) const { return Tok.is(Kind); } bool is(TokenType TT) const { return getType() == TT; } bool is(const IdentifierInfo *II) const { diff --git a/clang/lib/Format/MacroUnexpander.cpp b/clang/lib/Format/MacroUnexpander.cpp new file mode 100644 --- /dev/null +++ b/clang/lib/Format/MacroUnexpander.cpp @@ -0,0 +1,481 @@ +//===--- MacroUnexpander.cpp - Format C++ code ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the implementation of MacroUnexpander, which fits an +/// unexpanded macro call to a parsed set of UnwrappedLines. +/// +//===----------------------------------------------------------------------===// + +#include "Macros.h" + +#include "UnwrappedLineParser.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "format-unexpand" + +namespace clang { +namespace format { + +MacroUnexpander::MacroUnexpander( + unsigned Level, + const std::map> &Unexpanded) + : Level(Level), IdToUnexpanded(Unexpanded) { + Output.Tokens.push_back(std::make_unique()); + Lines.push_back(&Output); +} + +void MacroUnexpander::addLine(const UnwrappedLine &Line) { + assert(State != Finalized); + LLVM_DEBUG(llvm::dbgs() << "Unex: new line...\n"); + forEachToken(Line, [&](FormatToken *Token, FormatToken *Parent, bool First) { + add(Token, Parent, First); + }); +} + +UnwrappedLine MacroUnexpander::getResult() { + finalize(); + UnwrappedLine Final = + createUnwrappedLine(*Output.Tokens.front()->Children.front(), Level); + assert(!Final.Tokens.empty()); + return Final; +} + +// Call \p Call for each token in the unwrapped line given, passing +// the token, its parent and whether it is the first token in the line. +void MacroUnexpander::forEachToken( + const UnwrappedLine &Line, + const std::function &Call, + FormatToken *Parent) { + bool First = true; + for (const auto &N : Line.Tokens) { + Call(N.Tok, Parent, First); + First = false; + for (const auto &Child : N.Children) { + forEachToken(Child, Call, N.Tok); + } + } +} + +// Unexand \p Token, given its parent \p OriginalParent in the incoming +// unwrapped line. \p First specifies whether it is the first token in a given +// unwrapped line. +void MacroUnexpander::add(FormatToken *Token, FormatToken *OriginalParent, + bool First) { + LLVM_DEBUG( + llvm::dbgs() << "Unex: Token: " << Token->TokenText << ", Parent: " + << (OriginalParent ? OriginalParent->TokenText : "") + << ", First: " << First << "\n"); + // In order to be able to find the correct parent in the unexpanded token + // stream, we need to continue the unexpansion until we find the right token + // if it is part of the unexpanded token stream. + // Note that hidden tokens can be part of the unexpanded stream in nested + // macro calls. + // For example, given: BRACED(a) {a} + // And the call: BRACED(BRACED(x)) + // The outer macro call will be BRACED({a}), and the hidden tokens '{' and + // '}' can be found in the unexpanded macro stream of that level. + if (!Unexpanded.empty() && Token->MacroCtx && + (Token->MacroCtx->Role != MR_Hidden || + Unexpanded.size() != Token->MacroCtx->ExpandedFrom.size())) { + if (continueUnexpansionUntil(Token)) + First = true; + } + + prepareParent(OriginalParent, First); + + if (!Token->MacroCtx) { + // If this token was not generated by a macro call, we add it to the current + // line. + pushToken(Token); + } else { + // Otherwise add the unexpanded equivalent of the token. + unexpand(Token); + } +} + +// Ensures that: +// Lines.back() is the line that has \p OriginalParent or its unexpanded +// replacement token as a parent (when possible) - that is, the last token in +// \c Lines[Lines.size()-2] is the parent of Lines.back() in the +// unexpanded unwrapped line. +void MacroUnexpander::prepareParent(FormatToken *OriginalParent, bool First) { + // We want to find the parent in the new unwrapped line, where the original + // parent might have been replaced by an unxpansion. + FormatToken *Parent = getParentInOutput(OriginalParent); + LLVM_DEBUG(llvm::dbgs() << "Unex: New parent: " + << (Parent ? Parent->TokenText : "") << "\n"); + + FormatToken *OpenMacroParent = nullptr; + if (!MacroCallStructure.empty()) { + // Inside a macro expansion, it is possible to lose track of the correct + // parent - either because it is already popped, for example because it was + // in a different macro argument (e.g. M({, })), or when we work on invalid + // code. + // Thus, we use the innermost macro call's parent as the parent at which + // we stop; this allows us to stay within the macro expansion and keeps + // any problems confined to the extent of the macro call. + OpenMacroParent = + getParentInOutput(MacroCallStructure.back().RedirectParentTo); + } + if (First || (!Lines.back()->Tokens.empty() && + Parent == Lines.back()->Tokens.back()->Tok)) { + // If we are at the first token in a new line, we want to also + // create a new line in the resulting unexpanded unwrapped line. + while (Lines.back()->Tokens.empty() || + (Parent != Lines.back()->Tokens.back()->Tok && + Lines.back()->Tokens.back()->Tok != OpenMacroParent)) { + Lines.pop_back(); + assert(!Lines.empty()); + } + assert(!Lines.empty()); + Lines.back()->Tokens.back()->Children.push_back(std::make_unique()); + Lines.push_back(&*Lines.back()->Tokens.back()->Children.back()); + } else if (parentLine().Tokens.back()->Tok != Parent) { + // If we're not the first token in a new line, pop lines until we find + // the child of \c Parent in the stack. + while (Parent != parentLine().Tokens.back()->Tok && + parentLine().Tokens.back()->Tok && + parentLine().Tokens.back()->Tok != OpenMacroParent) { + Lines.pop_back(); + assert(!Lines.empty()); + } + } + assert(!Lines.empty()); +} + +// For a given \p Parent in the incoming expanded token stream, find the +// corresponding parent in the output. +FormatToken *MacroUnexpander::getParentInOutput(FormatToken *Parent) { + auto I = TokenToParentInOutput.find(Parent); + if (I == TokenToParentInOutput.end()) + return Parent; + for (; I != TokenToParentInOutput.end(); + I = TokenToParentInOutput.find(Parent)) { + Parent = I->second; + } + // If we use a different token than the parent in the expanded token stream + // as parent, mark it as a special parent, so the formatting code knows it + // needs to have its children formatted. + Parent->MacroParent = true; + return Parent; +} + +// Unexpand a \p Token that was expanded from a macro call. +void MacroUnexpander::unexpand(FormatToken *Token) { + assert(Token->MacroCtx); + // A single token can be the only result of a macro call: + // Given: ID(x, y) ; + // And the call: ID(, ) + // ';' in the expanded stream will unexpand all of ID(, ). + if (Token->MacroCtx->StartOfExpansion) { + startUnexpansion(Token); + // If the order of tokens in the expanded token stream is not the + // same as the order of tokens in the unexpanded stream, we need + // to unexpand tokens that arrive later in the stream. + if (Token->MacroCtx->Role != MR_Hidden) { + continueUnexpansionUntil(Token); + } + } + assert(!Unexpanded.empty()); + if (Unexpanded.back().I != Unexpanded.back().End) { + assert(Unexpanded.size() == Token->MacroCtx->ExpandedFrom.size()); + if (Token->MacroCtx->Role != MR_Hidden) { + // The current token in the unexpanded token stream must be the token + // we're looking for - we either arrive here after startUnexpansion, + // which initiates the stream to the first token, or after + // continueExpansionUntil skipped until the expected token in the + // unexpanded stream at the start of add(...). + assert(Unexpanded.back().I->Tok == Token); + processNextUnexpanded(); + } else if (!currentLine()->Tokens.empty()) { + // FIXME:assert(!currentLine()->Tokens.empty()); + // Map all hidden tokens to the last visible token in the output. + // If the hidden token is a parent, we'll use the last visible + // token as the parent of the hidden token's children. + TokenToParentInOutput[Token] = currentLine()->Tokens.back()->Tok; + } else { + for (auto I = Lines.rbegin(), E = Lines.rend(); I != E; ++I) { + if (!(*I)->Tokens.empty()) { + TokenToParentInOutput[Token] = (*I)->Tokens.back()->Tok; + + break; + } + } + llvm::dbgs() << "CURRENT LINE TOKENS EMPTY!!\n"; + } + } + if (Token->MacroCtx->EndOfExpansion) + endUnexpansion(Token); +} + +// Given a \p Token that starts an expansion, unexpand the beginning of the +// macro call. +// For example, given: ID(x) x +// And the call: ID(int a) +// Unexpands: ID( +void MacroUnexpander::startUnexpansion(FormatToken *Token) { + assert(Token->MacroCtx); + assert(!Token->MacroCtx->ExpandedFrom.empty()); + assert(Unexpanded.size() <= Token->MacroCtx->ExpandedFrom.size()); +#ifndef NDEBUG + // Check that the token's unexpansion stack matches our current unexpansion + // stack. + for (size_t I = 0; I < Unexpanded.size(); ++I) { + assert(Unexpanded[I].ID == + Token->MacroCtx + ->ExpandedFrom[Token->MacroCtx->ExpandedFrom.size() - 1 - I]); + } +#endif + // Start unexpansion for all calls for which this token is the first token + // generated by the call. + for (size_t I = Unexpanded.size(); I < Token->MacroCtx->ExpandedFrom.size(); + ++I) { + FormatToken *ID = + Token->MacroCtx + ->ExpandedFrom[Token->MacroCtx->ExpandedFrom.size() - 1 - I]; + // We found a macro call to be unexpanded; the next time our unexpansion + // stack is empty we know we finished an unexpansion. + State = InProgress; + // Put the unexpanded macro call's token into our unexpansion stack. + auto IU = IdToUnexpanded.find(ID); + assert(IU != IdToUnexpanded.end()); + Unexpanded.push_back( + {ID, IU->second->Tokens.begin(), IU->second->Tokens.end()}); + // Process the macro call's identifier. + processNextUnexpanded(); + if (Unexpanded.back().I == Unexpanded.back().End) + continue; + assert(Unexpanded.back().I->Tok->is(tok::l_paren)); + // Process the optional opening parenthesis. + processNextUnexpanded(); + } +} + +// Add all tokens in the unexpansion stream to the output until we find the +// given \p Token. +bool MacroUnexpander::continueUnexpansionUntil(FormatToken *Token) { + assert(!Unexpanded.empty()); + bool PassedMacroComma = false; + // FIXME: If Token was already expanded earlier, due to + // a change in order, we will not find it, but need to + // skip it. + for (; Unexpanded.back().I != Unexpanded.back().End && + Unexpanded.back().I->Tok != Token;) { + PassedMacroComma = processNextUnexpanded() || PassedMacroComma; + } + assert(Unexpanded.back().I == Unexpanded.back().End || + Unexpanded.back().I->Tok == Token); + return PassedMacroComma; +} + +// End all unexpansions for which \p Token is the final token. +void MacroUnexpander::endUnexpansion(FormatToken *Token) { + assert(!Token->MacroCtx || + (Unexpanded.size() >= Token->MacroCtx->EndOfExpansion)); + if (!Token->MacroCtx) + return; + for (size_t I = 0; I < Token->MacroCtx->EndOfExpansion; ++I) { +#ifndef NDEBUG + // Check all remaining tokens but the final closing parenthesis and optional + // trailing comment were already expanded at an inner expansion level. + for (auto T = Unexpanded.back().I; T != Unexpanded.back().End; ++T) { + FormatToken *Token = T->Tok; + bool ClosingParen = (std::next(T) == Unexpanded.back().End || + std::next(T)->Tok->isTrailingComment()) && + !Token->MacroCtx && Token->is(tok::r_paren); + bool TrailingComment = Token->isTrailingComment(); + bool PreviousLevel = + Token->MacroCtx && + (Unexpanded.size() < Token->MacroCtx->ExpandedFrom.size()); + if (!ClosingParen && !TrailingComment && !PreviousLevel) { + llvm::dbgs() << "At token: " << Token->TokenText << "\n"; + } + // In addition to the following cases, we can also run into this + // when a macro call had more arguments than expected; in that case, + // the comma and the remaining tokens in the macro call will potentially + // end up in the line when we finishe the expansion. + // FIXME: Add the information which arguments are unused, and assert + // one of the cases below plus unexpanded macro argument tokens. + // assert(ClosingParen || TrailingComment || PreviousLevel); + } +#endif + // Expand the closing parenthesis, if it exists, including an optional + // trailing comment. + for (auto T = Unexpanded.back().I; T != Unexpanded.back().End; ++T) { + processNextUnexpanded(); + } + Unexpanded.pop_back(); + } +} + +// If visible, add the next token of the unexpanded token sequence to the +// output. +bool MacroUnexpander::processNextUnexpanded() { + FormatToken *Token = Unexpanded.back().I->Tok; + // llvm::dbgs() << "ProcessNext: " << Token->TokenText << "\n"; + ++Unexpanded.back().I; + if (Token->MacroCtx) { + // Skip tokens that are not part of the macro call. + if (Token->MacroCtx->Role == MR_Hidden) { + return false; + } + // Skip tokens we already expanded during an inner unexpansion. + // For example, given: ID(x) {x} + // And the call: ID(ID(f)) + // We get two unexpansions: + // ID(f) -> {f} + // ID({f}) -> {{f}} + // We unexpand f during the first unexpansion, and skip it during the second + // unexpansion. + if (Unexpanded.size() < Token->MacroCtx->ExpandedFrom.size()) { + return false; + } + } + // Put the parentheses and commas of a macro call into the same line; + // if the arguments produce new unwrapped lines, they will become children + // of the corresponding opening parenthesis or comma tokens in the + // unexpanded call. + if (!Token->MacroCtx && Token->isOneOf(tok::comma, tok::r_paren) && + !MacroCallStructure.empty()) { + if (Token->is(tok::comma)) { + TokenToParentInOutput + [MacroCallStructure.back().Line->Tokens.back()->Tok] = Token; + Token->MacroParent = true; + pushToken(Token, MacroCallStructure.back().Line); + prepareParent(Token, /*First=*/true); + return true; + } + assert(Token->is(tok::r_paren)); + pushToken(Token, MacroCallStructure.back().Line); + TokenToParentInOutput.erase(MacroCallStructure.back().RedirectParentFrom); + MacroCallStructure.pop_back(); + if (!MacroCallStructure.empty()) { + // We might have overwritten the previous state's parent. + TokenToParentInOutput[MacroCallStructure.back().RedirectParentFrom] = + MacroCallStructure.back().RedirectParentTo; + } + return false; + } + if (!Token->MacroCtx && Token->is(tok::l_paren)) { + MacroCallStructure.push_back( + {currentLine(), parentLine().Tokens.back()->Tok, Token}); + TokenToParentInOutput[MacroCallStructure.back().RedirectParentFrom] = Token; + pushToken(Token); + prepareParent(Token, /*First=*/true); + Token->MacroParent = true; + return false; + } + // Note that any tokens that are tagged with MR_None have been passed as + // arguments to the macro that have not been expanded, for example: + // Given: ID(X) x + // When calling: ID(a, b) + // 'b' will be part of the unexpanded token stream, but tagged MR_None. + // Given that erroring out in this case would be disruptive, we continue + // pushing the (unformatted) token. + // FIXME: This can lead to unfortunate formatting decisions - give the user + // a hint that their macro definition is broken. + pushToken(Token); + return false; +} + +void MacroUnexpander::finalize() { + if (State == Finalized) + return; + assert(State == InProgress && Unexpanded.empty()); + State = Finalized; + + // We created corresponding unwrapped lines for each incoming line as children + // the the toplevel null token. + assert(Output.Tokens.size() == 1 && !Output.Tokens.front()->Children.empty()); + + // The first line becomes the top level line in the resuling unwrapped line. + auto *I = Output.Tokens.front()->Children.begin(); + ++I; + LLVM_DEBUG({ + llvm::dbgs() << "Finalizing unexpanded lines:\n"; + debug(Output, 0); + }); + for (auto *E = Output.Tokens.front()->Children.end(); I != E; ++I) { + if ((*I)->Tokens.empty()) + continue; + + // Every subsequent line will become a child of the last token in the + // previous line, which is the token prior to the first token in the line. + auto L = TokenToPrevious.find((*I)->Tokens.front()->Tok); + assert(L != TokenToPrevious.end()); + assert(L->second->Children.empty()); + L->second->Children.push_back(std::move(*I)); + + // Mark the previous line's last token as generated by a macro expansion + // so the formatting algorithm can take that into account. + L->second->Tok->MacroParent = true; + } + Output.Tokens.front()->Children.resize(1); +} + +void MacroUnexpander::pushToken(FormatToken *Token, Line *L) { + L = L ? L : currentLine(); + LLVM_DEBUG(llvm::dbgs() << "-> " << Token->TokenText << "\n"); + L->Tokens.push_back(std::make_unique(Token)); + if (PreviousNode != nullptr) { + assert(TokenToPrevious.find(Token) == TokenToPrevious.end()); + TokenToPrevious[Token] = PreviousNode; + } + PreviousNode = &*L->Tokens.back(); +} + +UnwrappedLine MacroUnexpander::createUnwrappedLine(const Line &Line, + int Level) { + UnwrappedLine Result; + Result.Level = Level; + for (const auto &N : Line.Tokens) { + Result.Tokens.push_back(N->Tok); + UnwrappedLineNode &Current = Result.Tokens.back(); + for (const auto &Child : N->Children) { + if (Child->Tokens.empty()) + continue; + Current.Children.push_back(createUnwrappedLine(*Child, Level + 1)); + } + if (Current.Children.size() == 1 && + Current.Tok->isOneOf(tok::l_paren, tok::comma)) { + Result.Tokens.splice(Result.Tokens.end(), + Current.Children.front().Tokens); + Current.Children.clear(); + } + } + return Result; +} + +void MacroUnexpander::debug(const Line &Line, int Level) { + for (int i = 0; i < Level; ++i) + llvm::dbgs() << " "; + for (const auto &N : Line.Tokens) { + if (!N) + continue; + if (N->Tok) + llvm::dbgs() << N->Tok->TokenText << " "; + for (const auto &Child : N->Children) { + llvm::dbgs() << "\n"; + debug(*Child, Level + 1); + for (int i = 0; i < Level; ++i) + llvm::dbgs() << " "; + } + } + llvm::dbgs() << "\n"; +} + +MacroUnexpander::Line &MacroUnexpander::parentLine() { + return **std::prev(std::prev(Lines.end())); +} + +MacroUnexpander::Line *MacroUnexpander::currentLine() { return Lines.back(); } + +} // namespace format +} // namespace clang \ No newline at end of file diff --git a/clang/lib/Format/Macros.h b/clang/lib/Format/Macros.h --- a/clang/lib/Format/Macros.h +++ b/clang/lib/Format/Macros.h @@ -37,26 +37,22 @@ #ifndef CLANG_LIB_FORMAT_MACROS_H #define CLANG_LIB_FORMAT_MACROS_H +#include +#include #include -#include #include -#include "Encoding.h" #include "FormatToken.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -namespace llvm { -class MemoryBuffer; -} // namespace llvm - namespace clang { -class IdentifierTable; -class SourceManager; - namespace format { -struct FormatStyle; + +struct UnwrappedLine; +struct UnwrappedLineNode; /// Takes a set of macro definitions as strings and allows expanding calls to /// those macros. @@ -135,6 +131,204 @@ llvm::StringMap Definitions; }; +/// Converts a stream of unwrapped lines of expanded tokens into unwrapped lines +/// containing the tokens of the macro call in a way that the resulting +/// unwrapped lines of the macro call best resemble the split of unwrapped lines +/// in the input. +/// +/// Given a mapping from the macro name identifier token in the macro call +/// to the tokens of the macro call, for example: +/// CLASSA -> CLASSA({public: void x();}) +/// +/// When getting the formatted lines of the expansion via the \c addLine method +/// (each '->' specifies a call to \c addLine ): +/// -> class A { +/// -> public: +/// -> void x(); +/// -> }; +/// +/// Creates the tree of unwrapped lines containing the macro call tokens so that +/// the macro call tokens fit the semantic structure of the expanded formatted +/// lines: +/// -> CLASSA({ +/// -> public: +/// -> void x(); +/// -> }) +class MacroUnexpander { +public: + /// Create an Unexpander whose resulting unexpanded line will start at + /// \p Level, unexpanding using the map from name identifier token + /// to the corresponding tokens of the original macro call. + MacroUnexpander(unsigned Level, + const std::map> + &Unexpanded); + + /// For the given \p Line, match all occurences of tokens expanded from a + /// macro to unwrapped lines in the original macro call so that the resulting + /// tree of unwrapped lines best resembles the structure of unwrapped lines + /// passed in via \c addLine. + void addLine(const UnwrappedLine &Line); + + /// Check whether at the current state there is no open macro expansion + /// that needs to be processed to finish an macro call. + bool finished() { return State == InProgress && Unexpanded.empty(); } + + /// Retrieve the formatted \c UnwrappedLine containing the orginal + /// macro calls, formatted according to the expanded token stream received + /// via \c addLine(). + /// Generally, this line tries to have the same structure as the expanded, + /// formatted unwrapped lines handed in via \c addLine(), with the exception + /// that for multiple top-level lines, each subsequent line will be the + /// child of the last token in its predecessor. + /// If a token in a macro argument is a child of a token in the expansion, + /// the parent will be the corresponding token in the macro call. + /// For example: + /// #define C(a, b) class C { a b + /// C(int x;, int y;) would expand to class C { int x; int y; where in a + /// formatted line "int x;" and "int y;" would both be new separate lines (at + /// different indentation levels). In the result, "int x;" will be a child of + /// the opening parenthesis in "C(" and "int y;" will be a child of the "," + /// token. + UnwrappedLine getResult(); + +private: + void forEachToken( + const UnwrappedLine &Line, + const std::function &Call, + FormatToken *Parent = nullptr); + void add(FormatToken *Token, FormatToken *OriginalParent, bool First); + void prepareParent(FormatToken *OriginalParent, bool First); + FormatToken *getParentInOutput(FormatToken *Parent); + void unexpand(FormatToken *Token); + void startUnexpansion(FormatToken *Token); + bool continueUnexpansionUntil(FormatToken *Token); + void endUnexpansion(FormatToken *Token); + bool processNextUnexpanded(); + void finalize(); + + struct Line; + + void pushToken(FormatToken *Token, Line *L = nullptr); + UnwrappedLine createUnwrappedLine(const Line &Line, int Level); + void debug(const Line &Line, int Level); + Line &parentLine(); + Line *currentLine(); + + enum UnexpanderState { + Start, // No macro expansion was found in the input yet. + InProgress, // During a macro unexpansion. + Finalized, // Past macro unexpansion, the result is finalized. + }; + UnexpanderState State = Start; + + // Node in which we build up the resulting unwrapped line; this type is + // analogous to UnwrappedLineNode. + struct LineNode { + LineNode() = default; + LineNode(FormatToken *Tok) : Tok(Tok) {} + FormatToken *Tok = nullptr; + llvm::SmallVector, 4> Children; + }; + + // Line in which we build up the resulting unwrapped line. + // FIXME: Investigate changing UnwrappedLine to a pointer type and using it + // instead of rolling our own type. + struct Line { + llvm::SmallVector, 4> Tokens; + }; + + // The line in which we collect the resulting unexpanded output. + // To reduce special cases in the algorithm, the first level of the line + // contains a single null token that has the unexpanded incoming + // lines as children. + // In the end, we stich the lines together so that each subsequent line + // is a child of the last token of the previous line. This is necessary + // in order to format the overall expression as a single logical line - + // if we created separate lines, we'd format them with their own top-level + // indent depending on the semantic structure, which is not desired. + Line Output; + + // Stack of currently "open" lines, where each line's predecessor's last + // token is the parent token for that line. + llvm::SmallVector Lines; + + // Maps from the original token to the token that takes its place in the + // unexpanded token stream in terms of parent-child relationships. + // Note that it might take multiple steps to arrive at the correct + // parent in the output. + // Given: C(a, b) []() { a; b; } + // And a call: C(f(), g()) + // The structure in the incoming formatted unwrapped line will be: + // []() { + // |- f(); + // \- g(); + // } + // with f and g being children of the opening brace. + // In the unexpanded call: + // C(f(), g()) + // \- f() + // \- g() + // We want f to be a child of the opening parenthesis and g to be a child + // of the comma token in the macro call. + // Thus, we map + // { -> ( + // and add + // ( -> , + // once we're past the comma in the unexpansion. + llvm::DenseMap TokenToParentInOutput; + + // Keeps track of a single expansion while we're unexpanding tokens it + // generated. + struct Expansion { + // The identifier token of the macro call. + FormatToken *ID; + // Our current position in the unexpansion. + std::list::iterator I; + // The end of the unexpanded token sequence. + std::list::iterator End; + }; + + // Stack of unexpanded macro calls for which we're in the middle + // of an expansion. + llvm::SmallVector Unexpanded; + + struct MacroCallState { + Line *Line; + FormatToken *RedirectParentFrom; + FormatToken *RedirectParentTo; + }; + + // Keeps track of the lines into which the opening brace/parenthesis & + // argument separating commas for each level in the macro call go in order to + // put the corresponding closing brace/parenthesis into the same line in the + // output and keep track of which parents in the expanded token stream map to + // which tokens in the unexpanded stream. + // When an opening brace/parenthesis has children, we want the structure of + // the output line to be: + // |- MACRO + // |- ( + // | \- + // |- , + // | \- + // \- ) + llvm::SmallVector MacroCallStructure; + + // For each token, the previous token in the resulting unexpanded token + // stream. + llvm::DenseMap TokenToPrevious; + + // The previous token's node we put into the resulting unexpanded token + // stream. + LineNode *PreviousNode = nullptr; + + // Level the generated UnwrappedLine will be at. + const unsigned Level; + + // Maps from identifier of the macro call to an unwrapped line containing + // all tokens of the macro call. + const std::map> &IdToUnexpanded; +}; + } // namespace format } // namespace clang diff --git a/clang/unittests/Format/CMakeLists.txt b/clang/unittests/Format/CMakeLists.txt --- a/clang/unittests/Format/CMakeLists.txt +++ b/clang/unittests/Format/CMakeLists.txt @@ -16,6 +16,7 @@ FormatTestTableGen.cpp FormatTestTextProto.cpp MacroExpanderTest.cpp + MacroUnexpanderTest.cpp NamespaceEndCommentsFixerTest.cpp SortImportsTestJS.cpp SortImportsTestJava.cpp diff --git a/clang/unittests/Format/MacroUnexpanderTest.cpp b/clang/unittests/Format/MacroUnexpanderTest.cpp new file mode 100644 --- /dev/null +++ b/clang/unittests/Format/MacroUnexpanderTest.cpp @@ -0,0 +1,650 @@ +#include "../../lib/Format/Macros.h" +#include "../../lib/Format/UnwrappedLineParser.h" +#include "TestLexer.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include +#include +#include + +namespace clang { +namespace format { +namespace { + +using UnexpandedMap = std::map>; + +// Keeps track of a sequence of macro expansions. +// +// The expanded tokens are accessible via getTokens(), while a map of macro call +// identifier token to unexpanded token stream is accessible via +// getUnexpanded(). +class Expansion { +public: + Expansion(TestLexer &Lex, MacroExpander &Macros) : Lex(Lex), Macros(Macros) {} + + // Appends the token stream obtained from expanding the macro Name given + // the provided arguments, to be later retrieved with getTokens(). + // Returns the list of tokens making up the unexpanded macro call. + TokenList + expand(llvm::StringRef Name, + const SmallVector, 1> &Args) { + auto *ID = Lex.id(Name); + auto UnexpandedLine = std::make_unique(); + UnexpandedLine->Tokens.push_back(ID); + if (!Args.empty()) { + UnexpandedLine->Tokens.push_back(Lex.id("(")); + for (auto I = Args.begin(), E = Args.end(); I != E; ++I) { + if (I != Args.begin()) + UnexpandedLine->Tokens.push_back(Lex.id(",")); + UnexpandedLine->Tokens.insert(UnexpandedLine->Tokens.end(), I->begin(), + I->end()); + } + UnexpandedLine->Tokens.push_back(Lex.id(")")); + } + Unexpanded[ID] = std::move(UnexpandedLine); + + auto Expanded = uneof(Macros.expand(ID, Args)); + Tokens.append(Expanded.begin(), Expanded.end()); + + TokenList UnexpandedTokens; + for (const UnwrappedLineNode &Node : Unexpanded[ID]->Tokens) { + UnexpandedTokens.push_back(Node.Tok); + } + return UnexpandedTokens; + } + + TokenList expand(llvm::StringRef Name, + const std::vector &Args = {}) { + return expand(Name, lexArgs(Args)); + } + + const UnexpandedMap &getUnexpanded() const { return Unexpanded; } + + const TokenList &getTokens() const { return Tokens; } + +private: + llvm::SmallVector + lexArgs(const std::vector &Args) { + llvm::SmallVector Result; + for (const auto &Arg : Args) { + Result.push_back(uneof(Lex.lex(Arg))); + } + return Result; + } + std::map> Unexpanded; + llvm::SmallVector Tokens; + TestLexer &Lex; + MacroExpander &Macros; +}; + +struct Chunk { + Chunk(llvm::ArrayRef Tokens) + : Tokens(Tokens.begin(), Tokens.end()) {} + Chunk(llvm::ArrayRef Children) + : Children(Children.begin(), Children.end()) {} + llvm::SmallVector Tokens; + llvm::SmallVector Children; +}; + +bool tokenMatches(const FormatToken *Left, const FormatToken *Right) { + if (Left->getType() == Right->getType() && + Left->TokenText == Right->TokenText) + return true; + llvm::dbgs() << Left->TokenText << " != " << Right->TokenText << "\n"; + return false; +} + +// Allows to produce chunks of a token list by typing the code of equal tokens. +// +// Created from a list of tokens, users call "consume" to get the next chunk +// of tokens, checking that they match the written code. +struct Matcher { + Matcher(const TokenList &Tokens, TestLexer &Lex) + : Tokens(Tokens), It(this->Tokens.begin()), Lex(Lex) {} + + Chunk consume(StringRef Tokens) { + TokenList Result; + for (const FormatToken *Token : uneof(Lex.lex(Tokens))) { + assert(tokenMatches(*It, Token)); + Result.push_back(*It); + ++It; + } + return Chunk(Result); + } + + TokenList Tokens; + TokenList::iterator It; + TestLexer &Lex; +}; + +UnexpandedMap mergeUnexpanded(const UnexpandedMap &M1, + const UnexpandedMap &M2) { + UnexpandedMap Result; + for (const auto &KV : M1) { + Result[KV.first] = std::make_unique(*KV.second); + } + for (const auto &KV : M2) { + Result[KV.first] = std::make_unique(*KV.second); + } + return Result; +} + +class MacroUnexpanderTest : public ::testing::Test { +public: + MacroUnexpanderTest() : Lex(Allocator, Buffers) {} + + std::unique_ptr + createExpander(const std::vector &MacroDefinitions) { + return std::make_unique(MacroDefinitions, + Lex.SourceMgr.get(), Lex.Style, + Lex.Allocator, Lex.IdentTable); + } + + UnwrappedLine line(llvm::ArrayRef Tokens) { + UnwrappedLine Result; + for (FormatToken *Tok : Tokens) { + Result.Tokens.push_back(UnwrappedLineNode(Tok)); + } + return Result; + } + + UnwrappedLine line(llvm::StringRef Text) { return line({lex(Text)}); } + + UnwrappedLine line(llvm::ArrayRef Chunks) { + UnwrappedLine Result; + for (const Chunk &Chunk : Chunks) { + Result.Tokens.insert(Result.Tokens.end(), Chunk.Tokens.begin(), + Chunk.Tokens.end()); + assert(!Result.Tokens.empty()); + Result.Tokens.back().Children.append(Chunk.Children.begin(), + Chunk.Children.end()); + } + return Result; + } + + TokenList lex(llvm::StringRef Text) { return uneof(Lex.lex(Text)); } + + Chunk tokens(llvm::StringRef Text) { return Chunk(lex(Text)); } + + Chunk children(llvm::ArrayRef Children) { + return Chunk(Children); + } + + llvm::SpecificBumpPtrAllocator Allocator; + std::vector> Buffers; + TestLexer Lex; +}; + +bool matchesTokens(const UnwrappedLine &L1, const UnwrappedLine &L2) { + if (L1.Tokens.size() != L2.Tokens.size()) + return false; + for (auto L1It = L1.Tokens.begin(), L2It = L2.Tokens.begin(); + L1It != L1.Tokens.end(); ++L1It, ++L2It) { + if (L1It->Tok != L2It->Tok) + return false; + if (L1It->Children.size() != L2It->Children.size()) + return false; + for (auto L1ChildIt = L1It->Children.begin(), + L2ChildIt = L2It->Children.begin(); + L1ChildIt != L1It->Children.end(); ++L1ChildIt, ++L2ChildIt) { + if (!matchesTokens(*L1ChildIt, *L2ChildIt)) + return false; + } + } + return true; +} +MATCHER_P(matchesLine, line, "") { return matchesTokens(arg, line); } + +TEST_F(MacroUnexpanderTest, Identifier) { + auto Macros = createExpander({"X=x"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("X"); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Unexp.addLine(line(Exp.getTokens())); + EXPECT_TRUE(Unexp.finished()); + UnwrappedLine Result = Unexp.getResult(); + Matcher U(Call, Lex); + EXPECT_THAT(Unexp.getResult(), matchesLine(line(U.consume("X")))); +} + +TEST_F(MacroUnexpanderTest, NestedLineWithinCall) { + auto Macros = createExpander({"C(a)=class C { a; };"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("C", {"void f()"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line(E.consume("class C {"))); + Unexp.addLine(line(E.consume("void f();"))); + Unexp.addLine(line(E.consume("};"))); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + EXPECT_THAT(Unexp.getResult(), matchesLine(line(U.consume("C(void f())")))); +} + +TEST_F(MacroUnexpanderTest, StatementSequence) { + auto Macros = createExpander({"SEMI=;"}); + Expansion Exp(Lex, *Macros); + TokenList Call1 = Exp.expand("SEMI"); + TokenList Call2 = Exp.expand("SEMI"); + TokenList Call3 = Exp.expand("SEMI"); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line(E.consume(";"))); + Unexp.addLine(line(E.consume(";"))); + Unexp.addLine(line(E.consume(";"))); + EXPECT_TRUE(Unexp.finished()); + Matcher U1(Call1, Lex); + Matcher U2(Call2, Lex); + Matcher U3(Call3, Lex); + EXPECT_THAT(Unexp.getResult(), + matchesLine(line( + {U1.consume("SEMI"), + children({line({U2.consume("SEMI"), + children({line(U3.consume("SEMI"))})})})}))); +} + +TEST_F(MacroUnexpanderTest, NestedBlock) { + auto Macros = createExpander({"ID(x)=x"}); + // Test: ID({ ID(a *b); }) + // 1. expand ID(a *b) -> a *b + Expansion Exp1(Lex, *Macros); + TokenList Call1 = Exp1.expand("ID", {"a *b"}); + // 2. expand ID({ a *b; }) + TokenList Arg; + Arg.push_back(Lex.id("{")); + Arg.append(Exp1.getTokens().begin(), Exp1.getTokens().end()); + Arg.push_back(Lex.id(";")); + Arg.push_back(Lex.id("}")); + Expansion Exp2(Lex, *Macros); + TokenList Call2 = Exp2.expand("ID", {Arg}); + + // Consume as-if formatted: + // { + // a *b; + // } + UnexpandedMap Unexpanded = + mergeUnexpanded(Exp1.getUnexpanded(), Exp2.getUnexpanded()); + MacroUnexpander Unexp(0, Unexpanded); + Matcher E(Exp2.getTokens(), Lex); + Unexp.addLine(line(E.consume("{"))); + Unexp.addLine(line(E.consume("a *b;"))); + Unexp.addLine(line(E.consume("}"))); + EXPECT_TRUE(Unexp.finished()); + + // Expect lines: + // ID({ + // ID(a *b); + // }) + Matcher U1(Call1, Lex); + Matcher U2(Call2, Lex); + auto Chunk2Start = U2.consume("ID("); + auto Chunk2LBrace = U2.consume("{"); + U2.consume("a *b"); + auto Chunk2Mid = U2.consume(";"); + auto Chunk2RBrace = U2.consume("}"); + auto Chunk2End = U2.consume(")"); + auto Chunk1 = U1.consume("ID(a *b)"); + + auto Expected = line({Chunk2Start, + children({ + line(Chunk2LBrace), + line({Chunk1, Chunk2Mid}), + line(Chunk2RBrace), + }), + Chunk2End}); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, NestedChildBlocks) { + auto Macros = createExpander({"ID(x)=x", "CALL(x)=f([] { x })"}); + // Test: ID(CALL(CALL(return a * b;))) + // 1. expand CALL(return a * b;) + Expansion Exp1(Lex, *Macros); + TokenList Call1 = Exp1.expand("CALL", {"return a * b;"}); + // 2. expand CALL(f([] { return a * b; })) + Expansion Exp2(Lex, *Macros); + TokenList Call2 = Exp2.expand("CALL", {Exp1.getTokens()}); + // 3. expand ID({ f([] { f([] { return a * b; }) }) }) + TokenList Arg3; + Arg3.push_back(Lex.id("{")); + Arg3.append(Exp2.getTokens().begin(), Exp2.getTokens().end()); + Arg3.push_back(Lex.id("}")); + Expansion Exp3(Lex, *Macros); + TokenList Call3 = Exp3.expand("ID", {Arg3}); + + // Consume as-if formatted in three unwrapped lines: + // 0: { + // 1: f([] { + // f([] { + // return a * b; + // }) + // }) + // 2: } + UnexpandedMap Unexpanded = mergeUnexpanded( + Exp1.getUnexpanded(), + mergeUnexpanded(Exp2.getUnexpanded(), Exp3.getUnexpanded())); + MacroUnexpander Unexp(0, Unexpanded); + Matcher E(Exp3.getTokens(), Lex); + Unexp.addLine(line(E.consume("{"))); + Unexp.addLine( + line({E.consume("f([] {"), + children({line({E.consume("f([] {"), + children({line(E.consume("return a * b;"))}), + E.consume("})")})}), + E.consume("})")})); + Unexp.addLine(line(E.consume("}"))); + EXPECT_TRUE(Unexp.finished()); + + // Expect lines: + // ID( + // { + // CALL(CALL(return a * b;)) + // } + // ) + Matcher U1(Call1, Lex); + Matcher U2(Call2, Lex); + Matcher U3(Call3, Lex); + auto Chunk3Start = U3.consume("ID("); + auto Chunk3LBrace = U3.consume("{"); + U3.consume("f([] { f([] { return a * b; }) })"); + auto Chunk3RBrace = U3.consume("}"); + auto Chunk3End = U3.consume(")"); + auto Chunk2Start = U2.consume("CALL("); + U2.consume("f([] { return a * b; })"); + auto Chunk2End = U2.consume(")"); + auto Chunk1 = U1.consume("CALL(return a * b;)"); + + auto Expected = line({ + Chunk3Start, + children({ + line(Chunk3LBrace), + line({ + Chunk2Start, + Chunk1, + Chunk2End, + }), + line(Chunk3RBrace), + }), + Chunk3End, + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, NestedChildrenMultipleArguments) { + auto Macros = createExpander({"CALL(a, b)=f([] { a; b; })"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("CALL", {std::string("int a"), "int b"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line({ + E.consume("f([] {"), + children({ + line(E.consume("int a;")), + line(E.consume("int b;")), + }), + E.consume("})"), + })); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line(U.consume("CALL(int a, int b)")); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, ReverseOrderArgumentsInExpansion) { + auto Macros = createExpander({"CALL(a, b)=b + a"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("CALL", {std::string("x"), "y"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line(E.consume("y + x"))); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line(U.consume("CALL(x, y)")); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, MultipleToplevelUnwrappedLines) { + auto Macros = createExpander({"ID(a, b)=a b"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("ID", {std::string("x; x"), "y"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line(E.consume("x;"))); + Unexp.addLine(line(E.consume("x y"))); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line({ + U.consume("ID("), + children({ + line(U.consume("x;")), + line(U.consume("x")), + }), + U.consume(", y)"), + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, NestedCallsMultipleLines) { + auto Macros = createExpander({"ID(x)=x"}); + // Test: ID({ID(a * b);}) + // 1. expand ID(a * b) + Expansion Exp1(Lex, *Macros); + TokenList Call1 = Exp1.expand("ID", {"a * b"}); + // 2. expand ID({ a * b; }) + Expansion Exp2(Lex, *Macros); + TokenList Arg2; + Arg2.push_back(Lex.id("{")); + Arg2.append(Exp1.getTokens().begin(), Exp1.getTokens().end()); + Arg2.push_back(Lex.id(";")); + Arg2.push_back(Lex.id("}")); + TokenList Call2 = Exp2.expand("ID", {Arg2}); + + // Consume as-if formatted in three unwrapped lines: + // 0: { + // 1: a * b; + // 2: } + UnexpandedMap Unexpanded = + mergeUnexpanded(Exp1.getUnexpanded(), Exp2.getUnexpanded()); + MacroUnexpander Unexp(0, Unexpanded); + Matcher E(Exp2.getTokens(), Lex); + Unexp.addLine(line(E.consume("{"))); + Unexp.addLine(line(E.consume("a * b;"))); + Unexp.addLine(line(E.consume("}"))); + EXPECT_TRUE(Unexp.finished()); + + // Expect lines: + // ID( + // { + // ID(a * b); + // } + // ) + Matcher U1(Call1, Lex); + Matcher U2(Call2, Lex); + auto Chunk2Start = U2.consume("ID("); + auto Chunk2LBrace = U2.consume("{"); + U2.consume("a * b"); + auto Chunk2Semi = U2.consume(";"); + auto Chunk2RBrace = U2.consume("}"); + auto Chunk2End = U2.consume(")"); + auto Chunk1 = U1.consume("ID(a * b)"); + + auto Expected = line({ + Chunk2Start, + children({ + line({Chunk2LBrace}), + line({Chunk1, Chunk2Semi}), + line({Chunk2RBrace}), + }), + Chunk2End, + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, ParentOutsideMacroCall) { + auto Macros = createExpander({"ID(a)=a"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("ID", {std::string("x; y; z;")}); + + auto Prefix = tokens("int a = []() {"); + auto Postfix = tokens("}();"); + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line({ + Prefix, + children({ + line(E.consume("x;")), + line(E.consume("y;")), + line(E.consume("z;")), + }), + Postfix, + })); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line({ + Prefix, + children({ + line({ + U.consume("ID("), + children({ + line(U.consume("x;")), + line(U.consume("y;")), + line(U.consume("z;")), + }), + U.consume(")"), + }), + }), + Postfix, + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, UnusedMacroArguments) { + auto Macros = createExpander({"X=x"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("X", {"a", "b", "c"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Unexp.addLine(line(Exp.getTokens())); + EXPECT_TRUE(Unexp.finished()); + UnwrappedLine Result = Unexp.getResult(); + Matcher U(Call, Lex); + EXPECT_THAT(Unexp.getResult(), matchesLine(line(U.consume("X(a, b, c)")))); +} + +TEST_F(MacroUnexpanderTest, UnusedEmptyMacroArgument) { + auto Macros = createExpander({"X=x"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("X", {std::string("")}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + auto Semi = tokens(";"); + Unexp.addLine(line({E.consume("x"), Semi})); + EXPECT_TRUE(Unexp.finished()); + UnwrappedLine Result = Unexp.getResult(); + Matcher U(Call, Lex); + EXPECT_THAT(Unexp.getResult(), matchesLine(line({U.consume("X()"), Semi}))); +} + +TEST_F(MacroUnexpanderTest, ChildrenSplitAcrossArguments) { + auto Macros = createExpander({"CALL(a, b)=f([]() a b)"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("CALL", {std::string("{ a;"), "b; }"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + Unexp.addLine(line({ + E.consume("f([]() {"), + children({ + line(E.consume("a;")), + line(E.consume("b;")), + }), + E.consume("})"), + })); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line({ + U.consume("CALL({"), + children(line(U.consume("a;"))), + U.consume(", b; })"), + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, ChildrenAfterMacroCall) { + auto Macros = createExpander({"CALL(a, b)=f([]() a b"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("CALL", {std::string("{ a"), "b"}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + auto Semi = tokens(";"); + auto SecondLine = tokens("c d;"); + auto ThirdLine = tokens("e f;"); + auto Postfix = tokens("})"); + Unexp.addLine(line({ + E.consume("f([]() {"), + children({ + line({E.consume("a b"), Semi}), + line(SecondLine), + line(ThirdLine), + }), + Postfix, + })); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line({ + U.consume("CALL({"), + children(line(U.consume("a"))), + U.consume(", b)"), + Semi, + children(line({ + SecondLine, + children(line({ + ThirdLine, + Postfix, + })), + })), + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +TEST_F(MacroUnexpanderTest, InvalidCodeSplittingBracesAcrossArgs) { + auto Macros = createExpander({"M(a, b)=(a) (b)"}); + Expansion Exp(Lex, *Macros); + TokenList Call = Exp.expand("M", {std::string("{"), "x", ""}); + + MacroUnexpander Unexp(0, Exp.getUnexpanded()); + Matcher E(Exp.getTokens(), Lex); + auto Prefix = tokens("({"); + Unexp.addLine(line({ + Prefix, + children({ + line({ + E.consume("({"), + children({line(E.consume(")(x)"))}), + }), + }), + })); + EXPECT_TRUE(Unexp.finished()); + Matcher U(Call, Lex); + auto Expected = line({ + Prefix, + children({line(U.consume("M({,x,)"))}), + }); + EXPECT_THAT(Unexp.getResult(), matchesLine(Expected)); +} + +} // namespace +} // namespace format +} // namespace clang