diff --git a/clang/lib/Format/CMakeLists.txt b/clang/lib/Format/CMakeLists.txt --- a/clang/lib/Format/CMakeLists.txt +++ b/clang/lib/Format/CMakeLists.txt @@ -7,6 +7,7 @@ Format.cpp FormatToken.cpp FormatTokenLexer.cpp + MacroExpander.cpp NamespaceEndCommentsFixer.cpp SortJavaScriptImports.cpp TokenAnalyzer.cpp diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -134,6 +134,68 @@ enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break }; +/// Roles a token can take in a configured macro expansion. +enum MacroRole { + /// The token was expanded from a macro argument when formatting the expanded + /// token sequence. + MR_ExpandedArg, + /// The token is part of a macro argument that was previously formatted as + /// expansion when formatting the unexpanded macro call. + MR_UnexpandedArg, + /// The token was expanded from a macro definition, and is not visible as part + /// of the macro call. + MR_Hidden, +}; + +struct FormatToken; + +/// Contains information on the token's role in a macro expansion. +/// +/// Given the following definitions: +/// A(X) = [ X ] +/// B(X) = < X > +/// C(X) = X +/// +/// Consider the macro call: +/// A({B(C(C(x)))}) -> [{}] +/// +/// In this case, the tokens of the unexpanded macro call will have the +/// following relevant entries in their macro context (note that formatting +/// the unexpanded macro call happens *after* formatting the expanded macro +/// call): +/// A( { B( C( C(x) ) ) } ) +/// Role: NN U NN NN NNUN N N U N (N=None, U=UnexpandedArg) +/// +/// [ { < x > } ] +/// Role: H E H E H E H (H=Hidden, E=ExpandedArg) +/// ExpandedFrom[0]: A A A A A A A +/// ExpandedFrom[1]: B B B +/// ExpandedFrom[2]: C +/// ExpandedFrom[3]: C +/// StartOfExpansion: 1 0 1 2 0 0 0 +/// EndOfExpansion: 0 0 0 2 1 0 1 +struct MacroContext { + MacroContext(MacroRole Role) : Role(Role) {} + + /// The token's role in the macro expansion. + /// When formatting an expanded macro, all tokens that are part of macro + /// arguments will be MR_ExpandedArg, while all tokens that are not visible in + /// the macro call will be MR_Hidden. + /// When formatting an unexpanded macro call, all tokens that are part of + /// macro arguments will be MR_UnexpandedArg. + MacroRole Role; + + /// The stack of macro call identifier tokens this token was expanded from. + llvm::SmallVector ExpandedFrom; + + /// The number of expansions of which this macro is the first entry. + unsigned StartOfExpansion = 0; + + /// The number of currently open expansions in \c ExpandedFrom this macro is + /// the last token in. + unsigned EndOfExpansion = 0; +}; + class TokenRole; class AnnotatedLine; @@ -228,7 +290,9 @@ /// A token can have a special role that can carry extra information /// about the token's formatting. - std::unique_ptr Role; + /// FIXME: Make FormatToken for parsing and AnnotatedToken two different + /// classes and make this a unique_ptr in the AnnotatedToken class. + std::shared_ptr Role; /// If this is an opening parenthesis, how are the parameters packed? ParameterPackingKind PackingKind = PPK_Inconclusive; @@ -331,6 +395,10 @@ /// changes. bool Finalized = false; + // Contains all attributes related to how this token takes part + // in a configured macro expansion. + llvm::Optional MacroCtx; + bool is(tok::TokenKind Kind) const { return Tok.is(Kind); } bool is(TokenType TT) const { return Type == TT; } bool is(const IdentifierInfo *II) const { @@ -572,10 +640,12 @@ : nullptr; } + void copyFrom(const FormatToken &Tok) { *this = Tok; } + private: - // Disallow copying. + // Only allow copying via the explicit copyInto method. FormatToken(const FormatToken &) = delete; - void operator=(const FormatToken &) = delete; + FormatToken &operator=(const FormatToken &) = default; template bool startsSequenceInternal(A K1, Ts... Tokens) const { diff --git a/clang/lib/Format/MacroExpander.cpp b/clang/lib/Format/MacroExpander.cpp new file mode 100644 --- /dev/null +++ b/clang/lib/Format/MacroExpander.cpp @@ -0,0 +1,213 @@ +//===--- MacroExpander.h - Format C++ code ----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the implementation of MacroExpander, which handles macro +/// configuration and expansion while formatting. +/// +//===----------------------------------------------------------------------===// + +#include "Macros.h" + +#include "Encoding.h" +#include "FormatToken.h" +#include "FormatTokenLexer.h" +#include "clang/Basic/TokenKinds.h" +#include "clang/Format/Format.h" +#include "clang/Lex/HeaderSearch.h" +#include "clang/Lex/HeaderSearchOptions.h" +#include "clang/Lex/Lexer.h" +#include "clang/Lex/ModuleLoader.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "llvm/Support/ErrorHandling.h" + +namespace clang { +namespace format { + +struct MacroExpander::Definition { + StringRef Name; + SmallVector Params; + SmallVector Body; + + // Map from each argument's name to its position in the argument list. + // With "M(x, y) x + y": + // x -> 0 + // y -> 1 + llvm::StringMap ArgMap; +}; + +class MacroExpander::DefinitionParser { +public: + DefinitionParser(ArrayRef Tokens) : Tokens(Tokens) { + assert(!Tokens.empty()); + Current = Tokens[0]; + } + + // Parse the token stream and return the corresonding Definition object. + // Returns an empty definition object with a null-Name on error. + MacroExpander::Definition parse() { + if (!Current->is(tok::identifier)) + return {}; + Def.Name = Current->TokenText; + nextToken(); + if (Current->is(tok::l_paren)) { + if (!parseParams()) + return {}; + } + if (!parseExpansion()) + return {}; + + return Def; + } + +private: + bool parseParams() { + assert(Current->is(tok::l_paren)); + nextToken(); + while (Current->is(tok::identifier)) { + Def.Params.push_back(Current); + Def.ArgMap[Def.Params.back()->TokenText] = Def.Params.size() - 1; + nextToken(); + if (Current->isNot(tok::comma)) + break; + nextToken(); + } + if (Current->isNot(tok::r_paren)) + return false; + nextToken(); + return true; + } + + bool parseExpansion() { + if (!Current->isOneOf(tok::equal, tok::eof)) + return false; + if (Current->is(tok::equal)) + nextToken(); + parseTail(); + return true; + } + + void parseTail() { + while (Current->isNot(tok::eof)) { + Def.Body.push_back(Current); + nextToken(); + } + Def.Body.push_back(Current); + } + + void nextToken() { + if (Pos + 1 < Tokens.size()) + ++Pos; + Current = Tokens[Pos]; + Current->Finalized = true; + } + + size_t Pos = 0; + FormatToken *Current = nullptr; + Definition Def; + ArrayRef Tokens; +}; + +MacroExpander::MacroExpander( + const std::vector &Macros, clang::SourceManager &SourceMgr, + const FormatStyle &Style, + llvm::SpecificBumpPtrAllocator &Allocator, + IdentifierTable &IdentTable) + : SourceMgr(SourceMgr), Style(Style), Allocator(Allocator), + IdentTable(IdentTable) { + parseDefinitions(Macros); +} + +MacroExpander::~MacroExpander() = default; + +void MacroExpander::parseDefinitions(const std::vector &Macros) { + for (const std::string &Macro : Macros) { + Buffers.push_back( + llvm::MemoryBuffer::getMemBufferCopy(Macro, "")); + clang::FileID FID = + SourceMgr.createFileID(SourceManager::Unowned, Buffers.back().get()); + FormatTokenLexer Lex(SourceMgr, FID, 0, Style, encoding::Encoding_UTF8, + Allocator, IdentTable); + const auto Tokens = Lex.lex(); + if (!Tokens.empty()) { + DefinitionParser Parser(Tokens); + auto Definition = Parser.parse(); + Definitions[Definition.Name] = Definition; + } + } +} + +bool MacroExpander::defined(llvm::StringRef Name) const { + return Definitions.find(Name) != Definitions.end(); +} + +llvm::SmallVector MacroExpander::expand(FormatToken *ID, + ArgsList Args) const { + assert(defined(ID->TokenText)); + SmallVector Result; + const Definition &Def = Definitions.lookup(ID->TokenText); + + // Adds the given token to Result. + auto pushToken = [&](FormatToken *Tok) { + Tok->MacroCtx->ExpandedFrom.push_back(ID); + Result.push_back(Tok); + }; + + // If Tok references a parameter, adds the corresponding argument to Result. + // Returns false if Tok does not reference a parameter. + auto expandArgument = [&](FormatToken *Tok) -> bool { + // If the current token references a parameter, expand the corresponding + // argument. + if (!Tok->is(tok::identifier)) + return false; + auto I = Def.ArgMap.find(Tok->TokenText); + if (I == Def.ArgMap.end()) + return false; + // If there are fewer arguments than referenced parameters, treat the + // parameter as empty. + // FIXME: Potentially fully abort the expansion instead. + if (I->getValue() >= Args.size()) + return true; + for (const auto &Tok : Args[I->getValue()]) { + // A token can be part of multiple macro arguments. + // For example, with "ID(x) x": + // in ID(ID(x)), 'x' is expanded first as argument to the inner + // ID, then again as argument to the outer ID. We keep the macro + // role the token had from the inner expansion. + if (!Tok->MacroCtx) + Tok->MacroCtx = MacroContext(MR_ExpandedArg); + pushToken(Tok); + } + return true; + }; + + // Expand the definition into Result. + for (FormatToken *Tok : Def.Body) { + if (expandArgument(Tok)) + continue; + // Create a copy of the tokens from the macro body, i.e. were not provided + // by user code. + FormatToken *New = new (Allocator.Allocate()) FormatToken; + New->copyFrom(*Tok); + assert(!New->MacroCtx); + // Tokens that are not part of the user code are not formatted. + New->MacroCtx = MacroContext(MR_Hidden); + pushToken(New); + } + assert(Result.size() >= 1 && Result.back()->is(tok::eof)); + if (Result.size() > 1) { + ++Result[0]->MacroCtx->StartOfExpansion; + ++Result[Result.size() - 2]->MacroCtx->EndOfExpansion; + } + return Result; +} + +} // namespace format +} // namespace clang diff --git a/clang/lib/Format/Macros.h b/clang/lib/Format/Macros.h new file mode 100644 --- /dev/null +++ b/clang/lib/Format/Macros.h @@ -0,0 +1,132 @@ +//===--- MacroExpander.h - Format C++ code ----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the main building blocks of macro support in +/// clang-format. +/// +/// In order to not violate the requirement that clang-format can format files +/// in isolation, clang-format's macro support uses expansions users provide +/// as part of clang-format's style configuration. +/// +/// Macro definitions are of the form "MACRO(p1, p2)=p1 + p2", but only support +/// one level of expansion (\see MacroExpander for a full description of what +/// is supported). +/// +/// As part of parsing, clang-format uses the MacroExpander to expand the +/// spelled token streams into expanded token streams when it encounters a +/// macro call. The UnwrappedLineParser continues to parse UnwrappedLines +/// from the expanded token stream. +/// After the expanded unwrapped lines are parsed, the MacroUnexpander matches +/// the spelled token stream into unwrapped lines that best resemble the +/// structure of the expanded unwrapped lines. +/// +/// When formatting, clang-format formats the expanded unwrapped lines first, +/// determining the token types. Next, it formats the spelled unwrapped lines, +/// keeping the token types fixed, while allowing other formatting decisions +/// to change. +/// +//===----------------------------------------------------------------------===// + +#ifndef CLANG_LIB_FORMAT_MACRO_EXPANDER_H +#define CLANG_LIB_FORMAT_MACRO_EXPANDER_H + +#include +#include +#include + +#include "Encoding.h" +#include "FormatToken.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +namespace llvm { +class MemoryBuffer; +} // namespace llvm + +namespace clang { +class IdentifierTable; +class SourceManager; + +namespace format { +struct FormatStyle; + +/// Takes a set of macro definitions as strings and allows expanding calls to +/// those macros. +/// +/// For example: +/// Defintion: A(x, y)=x + y +/// Call : A(int a = 1, 2) +/// Expansion: int a = 1 + 2 +/// +/// Expansion does not check arity of the definition. +/// If fewer arguments than expected are provided, the remaining parameters +/// are considered empty: +/// Call : A(a) +/// Expansion: a + +/// If more arguments than expected are provided, they will be discarded. +/// +/// The expander does not support: +/// - recursive expansion +/// - stringification +/// - concatenation +/// - variadic macros +/// +/// Furthermore, expansion and definition are independent - a macro defined +/// as "A()=a" and "A=a" can both be expanded as "A()" and "A". +class MacroExpander { +public: + using ArgsList = llvm::ArrayRef>; + + /// Construct a macro expander from a set of macro definitions. + /// Macro definitions must be encoded as UTF-8. + /// + /// Each entry in \p Macros must conform to the following simple + /// macro-definition language: + /// ::= | "(" ")" + /// ::= | "" + /// ::= | "," + /// ::= "=" | + /// ::= | + /// + /// Macros that cannot be parsed will be silently discarded. + /// + MacroExpander(const std::vector &Macros, + clang::SourceManager &SourceMgr, const FormatStyle &Style, + llvm::SpecificBumpPtrAllocator &Allocator, + IdentifierTable &IdentTable); + ~MacroExpander(); + + /// Returns whether a macro \p Name is defined. + bool defined(llvm::StringRef Name) const; + + /// Returns the expanded stream of format tokens for \p ID, where + /// each element in \p Args is a positional argument to the macro call. + llvm::SmallVector expand(FormatToken *ID, + ArgsList Args) const; + +private: + struct Definition; + class DefinitionParser; + + void parseDefinitions(const std::vector &Macros); + + clang::SourceManager &SourceMgr; + const FormatStyle &Style; + llvm::SpecificBumpPtrAllocator &Allocator; + IdentifierTable &IdentTable; + std::vector> Buffers; + llvm::StringMap Definitions; +}; + +} // namespace format +} // namespace clang + +#endif diff --git a/clang/unittests/Format/CMakeLists.txt b/clang/unittests/Format/CMakeLists.txt --- a/clang/unittests/Format/CMakeLists.txt +++ b/clang/unittests/Format/CMakeLists.txt @@ -15,6 +15,7 @@ FormatTestSelective.cpp FormatTestTableGen.cpp FormatTestTextProto.cpp + MacroExpanderTest.cpp NamespaceEndCommentsFixerTest.cpp SortImportsTestJS.cpp SortImportsTestJava.cpp diff --git a/clang/unittests/Format/MacroExpanderTest.cpp b/clang/unittests/Format/MacroExpanderTest.cpp new file mode 100644 --- /dev/null +++ b/clang/unittests/Format/MacroExpanderTest.cpp @@ -0,0 +1,170 @@ +#include "../../lib/Format/Macros.h" +#include "TestLexer.h" +#include "clang/Basic/FileManager.h" + +#include "gtest/gtest.h" + +namespace clang { +namespace format { + +namespace { + +class MacroExpanderTest : public ::testing::Test { +public: + std::unique_ptr + create(const std::vector &MacroDefinitions) { + return std::make_unique(MacroDefinitions, + Lex.SourceMgr.get(), Lex.Style, + Lex.Allocator, Lex.IdentTable); + } + + std::string expand(MacroExpander &Macros, llvm::StringRef Name, + const std::vector &Args = {}) { + EXPECT_TRUE(Macros.defined(Name)); + return text(Macros.expand(Lex.id(Name), lexArgs(Args))); + } + + llvm::SmallVector + lexArgs(const std::vector &Args) { + llvm::SmallVector Result; + for (const auto &Arg : Args) { + Result.push_back(uneof(Lex.lex(Arg))); + } + return Result; + } + + struct MacroAttributes { + clang::tok::TokenKind Kind; + MacroRole Role; + unsigned Start; + unsigned End; + llvm::SmallVector ExpandedFrom; + }; + + void expectAttributes(const TokenList &Tokens, + const std::vector &Attributes, + const std::string &File, unsigned Line) { + EXPECT_EQ(Tokens.size(), Attributes.size()) << text(Tokens); + for (size_t I = 0, E = Tokens.size(); I != E; ++I) { + if (I >= Attributes.size()) + continue; + std::string Context = + ("for token " + llvm::Twine(I) + ": " + Tokens[I]->Tok.getName() + + " / " + Tokens[I]->TokenText) + .str(); + EXPECT_TRUE(Tokens[I]->is(Attributes[I].Kind)) + << Context << " in " << text(Tokens) << " at " << File << ":" << Line; + EXPECT_EQ(Tokens[I]->MacroCtx->Role, Attributes[I].Role) + << Context << " in " << text(Tokens) << " at " << File << ":" << Line; + EXPECT_EQ(Tokens[I]->MacroCtx->StartOfExpansion, Attributes[I].Start) + << Context << " in " << text(Tokens) << " at " << File << ":" << Line; + EXPECT_EQ(Tokens[I]->MacroCtx->EndOfExpansion, Attributes[I].End) + << Context << " in " << text(Tokens) << " at " << File << ":" << Line; + EXPECT_EQ(Tokens[I]->MacroCtx->ExpandedFrom, Attributes[I].ExpandedFrom) + << Context << " in " << text(Tokens) << " at " << File << ":" << Line; + } + } + + TestLexer Lex; +}; + +#define EXPECT_ATTRIBUTES(Tokens, Attributes) \ + expectAttributes(Tokens, Attributes, __FILE__, __LINE__) + +TEST_F(MacroExpanderTest, SkipsDefinitionOnError) { + auto Macros = + create({"A(", "B(,", "C(a,", "D(a a", "E(a, a", "F(,)", "G(a;"}); + for (const auto *Name : {"A", "B", "C", "D", "E", "F", "G"}) { + EXPECT_FALSE(Macros->defined(Name)) << "for Name " << Name; + } +} + +TEST_F(MacroExpanderTest, ExpandsWithoutArguments) { + auto Macros = create({ + "A", + "B=b", + "C=c + c", + "D()", + }); + EXPECT_EQ("", expand(*Macros, "A")); + EXPECT_EQ("b", expand(*Macros, "B")); + EXPECT_EQ("c+c", expand(*Macros, "C")); + EXPECT_EQ("", expand(*Macros, "D")); +} + +TEST_F(MacroExpanderTest, ExpandsWithArguments) { + auto Macros = create({ + "A(x)", + "B(x, y)=x + y", + }); + EXPECT_EQ("", expand(*Macros, "A", {"a"})); + EXPECT_EQ("b1+b2+b3", expand(*Macros, "B", {"b1", "b2 + b3"})); + EXPECT_EQ("x+", expand(*Macros, "B", {"x"})); +} + +TEST_F(MacroExpanderTest, AttributizesTokens) { + auto Macros = create({ + "A(x, y)={ x + y; }", + "B(x, y)=x + 3 + y", + }); + auto *A = Lex.id("A"); + auto AArgs = lexArgs({"a1 * a2", "a3 * a4"}); + auto Result = Macros->expand(A, AArgs); + EXPECT_EQ(11U, Result.size()) << text(Result) << " / " << Result; + EXPECT_EQ("{a1*a2+a3*a4;}", text(Result)); + std::vector Attributes = { + {tok::l_brace, MR_Hidden, 1, 0, {A}}, + {tok::identifier, MR_ExpandedArg, 0, 0, {A}}, + {tok::star, MR_ExpandedArg, 0, 0, {A}}, + {tok::identifier, MR_ExpandedArg, 0, 0, {A}}, + {tok::plus, MR_Hidden, 0, 0, {A}}, + {tok::identifier, MR_ExpandedArg, 0, 0, {A}}, + {tok::star, MR_ExpandedArg, 0, 0, {A}}, + {tok::identifier, MR_ExpandedArg, 0, 0, {A}}, + {tok::semi, MR_Hidden, 0, 0, {A}}, + {tok::r_brace, MR_Hidden, 0, 1, {A}}, + {tok::eof, MR_Hidden, 0, 0, {A}}, + }; + EXPECT_ATTRIBUTES(Result, Attributes); + + auto *B = Lex.id("B"); + auto BArgs = lexArgs({"b1", "b2"}); + Result = Macros->expand(B, BArgs); + EXPECT_EQ(6U, Result.size()) << text(Result) << " / " << Result; + EXPECT_EQ("b1+3+b2", text(Result)); + Attributes = { + {tok::identifier, MR_ExpandedArg, 1, 0, {B}}, + {tok::plus, MR_Hidden, 0, 0, {B}}, + {tok::numeric_constant, MR_Hidden, 0, 0, {B}}, + {tok::plus, MR_Hidden, 0, 0, {B}}, + {tok::identifier, MR_ExpandedArg, 0, 1, {B}}, + {tok::eof, MR_Hidden, 0, 0, {B}}, + }; + EXPECT_ATTRIBUTES(Result, Attributes); +} + +TEST_F(MacroExpanderTest, RecursiveExpansion) { + auto Macros = create({ + "A(x)=x", + "B(x)=x", + "C(x)=x", + }); + + auto *A = Lex.id("A"); + auto *B = Lex.id("B"); + auto *C = Lex.id("C"); + + auto Args = lexArgs({"id"}); + auto CResult = uneof(Macros->expand(C, Args)); + auto BResult = uneof(Macros->expand(B, CResult)); + auto AResult = uneof(Macros->expand(A, BResult)); + + std::vector Attributes = { + {tok::identifier, MR_ExpandedArg, 3, 3, {C, B, A}}, + }; + EXPECT_ATTRIBUTES(AResult, Attributes); +} + +} // namespace +} // namespace format +} // namespace clang diff --git a/clang/unittests/Format/TestLexer.h b/clang/unittests/Format/TestLexer.h new file mode 100644 --- /dev/null +++ b/clang/unittests/Format/TestLexer.h @@ -0,0 +1,88 @@ +//===--- TestLexer.h - Format C++ code --------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains a TestLexer to create FormatTokens from strings. +/// +//===----------------------------------------------------------------------===// + +#ifndef CLANG_UNITTESTS_FORMAT_TEST_LEXER_H +#define CLANG_UNITTESTS_FORMAT_TEST_LEXER_H + +#include "../../lib/Format/FormatTokenLexer.h" + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/SourceManager.h" + +#include +#include + +namespace clang { +namespace format { + +typedef llvm::SmallVector TokenList; + +inline std::ostream &operator<<(std::ostream &Stream, const FormatToken &Tok) { + Stream << "(" << Tok.Tok.getName() << ", \"" << Tok.TokenText.str() << "\")"; + return Stream; +} +inline std::ostream &operator<<(std::ostream &Stream, const TokenList &Tokens) { + Stream << "{"; + for (size_t I = 0, E = Tokens.size(); I != E; ++I) { + Stream << (I > 0 ? ", " : "") << *Tokens[I]; + } + Stream << "}"; + return Stream; +} + +inline TokenList uneof(const TokenList &Tokens) { + assert(!Tokens.empty() && Tokens.back()->is(tok::eof)); + return TokenList(Tokens.begin(), std::prev(Tokens.end())); +} + +inline std::string text(llvm::ArrayRef Tokens) { + return std::accumulate(Tokens.begin(), Tokens.end(), std::string(), + [](const std::string &R, FormatToken *Tok) { + return (R + Tok->TokenText).str(); + }); +} + +class TestLexer { +public: + TestLexer() : SourceMgr("test.cpp", "") {} + + TokenList lex(llvm::StringRef Code) { + Buffers.push_back( + llvm::MemoryBuffer::getMemBufferCopy(Code, "")); + clang::FileID FID = SourceMgr.get().createFileID(SourceManager::Unowned, + Buffers.back().get()); + FormatTokenLexer Lex(SourceMgr.get(), FID, 0, Style, Encoding, Allocator, + IdentTable); + auto Result = Lex.lex(); + return TokenList(Result.begin(), Result.end()); + } + + FormatToken *id(llvm::StringRef Code) { + auto Result = uneof(lex(Code)); + assert(Result.size() == 1U && "Code must expand to 1 token."); + return Result[0]; + } + + FormatStyle Style = getLLVMStyle(); + encoding::Encoding Encoding = encoding::Encoding_UTF8; + std::vector> Buffers; + clang::SourceManagerForFile SourceMgr; + llvm::SpecificBumpPtrAllocator Allocator; + IdentifierTable IdentTable; +}; + +} // namespace format +} // namespace clang + +#endif // LLVM_CLANG_UNITTESTS_FORMAT_TEST_LEXER_H