diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt --- a/clang-tools-extra/clangd/CMakeLists.txt +++ b/clang-tools-extra/clangd/CMakeLists.txt @@ -145,6 +145,10 @@ $ ) +target_include_directories(clangDaemon PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../pseudo/include +) + clang_target_link_libraries(clangDaemon PRIVATE clangAST @@ -170,6 +174,8 @@ clangTidy clangdSupport + + clangPseudo ) if(CLANGD_TIDY_CHECKS) target_link_libraries(clangDaemon PRIVATE ${ALL_CLANG_TIDY_CHECKS}) diff --git a/clang-tools-extra/clangd/SemanticSelection.h b/clang-tools-extra/clangd/SemanticSelection.h --- a/clang-tools-extra/clangd/SemanticSelection.h +++ b/clang-tools-extra/clangd/SemanticSelection.h @@ -15,6 +15,7 @@ #include "ParsedAST.h" #include "Protocol.h" #include "llvm/Support/Error.h" +#include #include namespace clang { namespace clangd { @@ -29,6 +30,11 @@ /// This should include large scopes, preprocessor blocks etc. llvm::Expected> getFoldingRanges(ParsedAST &AST); +/// Returns a list of ranges whose contents might be collapsible in an editor. +/// This version uses the pseudoparser which does not require the AST. +llvm::Expected> +getFoldingRanges(const std::string &Code); + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/SemanticSelection.cpp b/clang-tools-extra/clangd/SemanticSelection.cpp --- a/clang-tools-extra/clangd/SemanticSelection.cpp +++ b/clang-tools-extra/clangd/SemanticSelection.cpp @@ -11,6 +11,9 @@ #include "Protocol.h" #include "Selection.h" #include "SourceCode.h" +#include "clang-pseudo/Bracket.h" +#include "clang-pseudo/DirectiveTree.h" +#include "clang-pseudo/Token.h" #include "clang/AST/DeclBase.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" @@ -162,5 +165,39 @@ return collectFoldingRanges(SyntaxTree, AST.getSourceManager()); } +llvm::Expected> +getFoldingRanges(const std::string &Code) { + auto OrigStream = clang::pseudo::lex(Code, clang::pseudo::genericLangOpts()); + + auto DirectiveStructure = clang::pseudo::DirectiveTree::parse(OrigStream); + clang::pseudo::chooseConditionalBranches(DirectiveStructure, OrigStream); + + llvm::Optional Preprocessed; + Preprocessed = DirectiveStructure.stripDirectives(OrigStream); + + auto ParseableStream = clang::pseudo::stripComments( + cook(*Preprocessed, clang::pseudo::genericLangOpts())); + pseudo::pairBrackets(ParseableStream); + + std::vector Result; + for (const auto &Tok : ParseableStream.tokens()) { + if (auto *Paired = Tok.pair()) { + if (Tok.Line < Paired->Line) { + Position Start = offsetToPosition( + Code, OrigStream.origToken(Tok).text().data() - Code.data()); + Position End = offsetToPosition( + Code, OrigStream.origToken(*Paired).text().data() - Code.data()); + FoldingRange FR; + FR.startLine = Start.line; + FR.startCharacter = Start.character + 1; + FR.endLine = End.line; + FR.endCharacter = End.character; + Result.push_back(FR); + } + } + } + return Result; +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp b/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp @@ -240,7 +240,7 @@ ]]} )cpp", R"cpp( - class Foo { + class Foo {[[ public: Foo() {[[ int X = 1; @@ -253,14 +253,25 @@ // Braces are located at the same line: no folding range here. void getFooBar() { } - }; + ]]}; + )cpp", + R"cpp( + // Range boundaries on escaped newlines. + class Foo \ + \ + {[[ \ + public: + Foo() {[[\ + int X = 1; + ]]} \ + ]]}; )cpp", }; for (const char *Test : Tests) { auto T = Annotations(Test); - auto AST = TestTU::withCode(T.code()).build(); - EXPECT_THAT(gatherFoldingRanges(llvm::cantFail(getFoldingRanges(AST))), - UnorderedElementsAreArray(T.ranges())) + EXPECT_THAT( + gatherFoldingRanges(llvm::cantFail(getFoldingRanges(T.code().str()))), + UnorderedElementsAreArray(T.ranges())) << Test; } } diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h --- a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h @@ -33,6 +33,7 @@ #include "clang/Basic/TokenKinds.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/Support/raw_ostream.h" +#include #include #include #include @@ -67,6 +68,8 @@ uint8_t Indent = 0; /// Flags have some meaning defined by the function that produced this stream. uint8_t Flags = 0; + /// Index into the original token stream (of the original source file). + Index OrigTokIdx = 0; // Helpers to get/set Flags based on `enum class`. template bool flag(T Mask) const { return Flags & uint8_t{static_cast>(Mask)}; @@ -96,7 +99,7 @@ /// If this token is a paired bracket, the offset of the pair in the stream. int32_t Pair = 0; }; -static_assert(sizeof(Token) <= sizeof(char *) + 20, "Careful with layout!"); +static_assert(sizeof(Token) <= sizeof(char *) + 24, "Careful with layout!"); llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Token &); /// A half-open range of tokens within a stream. @@ -170,6 +173,18 @@ return Storage[1]; } + /// An original token stream corresponds to the original source file (Eg. + /// produced by lex()). It is not derived from another token stream. + bool isOriginal() const { return IsOriginal; } + void setOriginal() { IsOriginal = true; } + + /// Extracts the token in original stream corresponding to Token T. + const Token &origToken(const Token &T) const { + assert(isOriginal() && "stream is derived"); + assert(T.OrigTokIdx < tokens().size() && "invalid index"); + return tokens()[T.OrigTokIdx]; + } + /// Print the tokens in this stream to the output stream. /// /// The presence of newlines/spaces is preserved, but not the quantity. @@ -180,6 +195,7 @@ MutableArrayRef Tokens; std::vector Storage; // eof + Tokens + eof + bool IsOriginal = false; }; llvm::raw_ostream &operator<<(llvm::raw_ostream &, const TokenStream &); diff --git a/clang-tools-extra/pseudo/lib/Lex.cpp b/clang-tools-extra/pseudo/lib/Lex.cpp --- a/clang-tools-extra/pseudo/lib/Lex.cpp +++ b/clang-tools-extra/pseudo/lib/Lex.cpp @@ -26,7 +26,10 @@ TokenStream Result; clang::Token CT; + // Index into the token stream of original source code. + unsigned TokenIdx = 0; unsigned LastOffset = 0; + unsigned LineStartOffset = 0; unsigned Line = 0; unsigned Indent = 0; for (Lexer.LexFromRawLexer(CT); CT.getKind() != clang::tok::eof; @@ -40,16 +43,17 @@ Tok.Kind = CT.getKind(); // Update current line number and indentation from raw source code. - unsigned NewLineStart = 0; + bool SawNewLine = 0; for (unsigned I = LastOffset; I < Offset; ++I) { if (Code[I] == '\n') { - NewLineStart = I + 1; + LineStartOffset = I + 1; + SawNewLine = true; ++Line; } } - if (NewLineStart || !LastOffset) { + if (SawNewLine || !LastOffset) { Indent = 0; - for (char C : StringRef(Code).slice(NewLineStart, Offset)) { + for (char C : StringRef(Code).slice(LineStartOffset, Offset)) { if (C == ' ') ++Indent; else if (C == '\t') @@ -66,9 +70,12 @@ if (CT.needsCleaning() || CT.hasUCN()) Tok.setFlag(LexFlags::NeedsCleaning); + Tok.OrigTokIdx = TokenIdx; Result.push(Tok); LastOffset = Offset; + TokenIdx++; } + Result.setOriginal(); Result.finalize(); return Result; }