diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp --- a/clang-tools-extra/clangd/CodeComplete.cpp +++ b/clang-tools-extra/clangd/CodeComplete.cpp @@ -1033,8 +1033,11 @@ PP.getIdentifierTable().getExternalIdentifierLookup(); if (!PreambleIdentifiers || !PreambleMacros) return; - for (const auto &MacroName : Preamble.MainFileMacros) - if (auto *II = PreambleIdentifiers->get(MacroName)) + llvm::StringSet<> MacroNames; + for (const auto &Macro : Preamble.MainFileMacros) + MacroNames.insert(Macro.Name); + for (const auto &MacroName : MacroNames) + if (auto *II = PreambleIdentifiers->get(MacroName.getKey())) if (II->isOutOfDate()) PreambleMacros->updateOutOfDateIdentifier(*II); } diff --git a/clang-tools-extra/clangd/Macro.h b/clang-tools-extra/clangd/Macro.h new file mode 100644 --- /dev/null +++ b/clang-tools-extra/clangd/Macro.h @@ -0,0 +1,75 @@ +//===--- Macro.h -------------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_MACRO_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_MACRO_H + +#include "AST.h" +#include "Protocol.h" +#include "SourceCode.h" +#include "index/SymbolID.h" +#include "clang/Lex/PPCallbacks.h" +#include + +namespace clang { +namespace clangd { + +// Repsents a macro reference (definition or expansion) in the main file. +struct MainFileMacro { + std::string Name; // The macro name. + Range R; // range for the macro. +}; + +// This collects macro definitions and expansions in the main file. it is used +// to: +// - collect macros in the preamble section of the main file (in Preamble.cpp) +// - collect macros after the preamble of the main file (in ParsedAST.cpp) +class CollectMainFileMacros : public PPCallbacks { +public: + explicit CollectMainFileMacros(const SourceManager &SM, + const LangOptions &LangOpts, + std::vector &Out) + : SM(SM), LangOpts(LangOpts), MainFileMacros(Out) {} + + void FileChanged(SourceLocation Loc, FileChangeReason, + SrcMgr::CharacteristicKind, FileID) override { + InMainFile = isInsideMainFile(Loc, SM); + } + + void MacroDefined(const Token &MacroName, const MacroDirective *MD) override { + Add(MacroName, MD->getMacroInfo()); + } + + void MacroExpands(const Token &MacroName, const MacroDefinition &MD, + SourceRange Range, const MacroArgs *Args) override { + Add(MacroName, MD.getMacroInfo()); + } + +private: + void Add(const Token &MacroNameTok, const MacroInfo *MI) { + if (!InMainFile) + return; + auto Loc = MacroNameTok.getLocation(); + if (Loc.isMacroID()) + return; + + if (auto Range = getTokenRange(SM, LangOpts, MacroNameTok.getLocation())) { + MainFileMacros.push_back( + {MacroNameTok.getIdentifierInfo()->getName(), *Range}); + } + } + const SourceManager &SM; + const LangOptions &LangOpts; + bool InMainFile = true; + std::vector &MainFileMacros; +}; + +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_MACRO_H \ No newline at end of file diff --git a/clang-tools-extra/clangd/ParsedAST.h b/clang-tools-extra/clangd/ParsedAST.h --- a/clang-tools-extra/clangd/ParsedAST.h +++ b/clang-tools-extra/clangd/ParsedAST.h @@ -24,6 +24,7 @@ #include "Compiler.h" #include "Diagnostics.h" #include "Headers.h" +#include "Macro.h" #include "Path.h" #include "Preamble.h" #include "index/CanonicalIncludes.h" @@ -89,10 +90,9 @@ const IncludeStructure &getIncludeStructure() const; const CanonicalIncludes &getCanonicalIncludes() const; - /// Gets all macro locations (definition, expansions) present in the main - /// file. - /// NOTE: macros inside the preamble are not included. - llvm::ArrayRef getMacros() const; + /// Gets all macro references (definition, expansions) present in the main + /// file, including those in the preamble region. + llvm::ArrayRef getMacros() const; /// Tokens recorded while parsing the main file. /// (!) does not have tokens from the preamble. const syntax::TokenBuffer &getTokens() const { return Tokens; } @@ -101,7 +101,7 @@ ParsedAST(std::shared_ptr Preamble, std::unique_ptr Clang, std::unique_ptr Action, syntax::TokenBuffer Tokens, - std::vector MainFileMacroExpLocs, + std::vector MainFileMacroExpLocs, std::vector LocalTopLevelDecls, std::vector Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes); @@ -121,10 +121,8 @@ /// - Does not have spelled or expanded tokens for files from preamble. syntax::TokenBuffer Tokens; - /// The start locations of all macro definitions/expansions spelled **after** - /// preamble. - /// Does not include locations from inside other macro expansions. - std::vector MacroIdentifierLocs; + /// All macro definitions/expansions in the main file. + std::vector MainFileMacros; // Data, stored after parsing. std::vector Diags; // Top-level decls inside the current file. Not that this does not include diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -15,6 +15,7 @@ #include "Headers.h" #include "IncludeFixer.h" #include "Logger.h" +#include "Macro.h" #include "SourceCode.h" #include "Trace.h" #include "index/CanonicalIncludes.h" @@ -98,33 +99,6 @@ std::vector TopLevelDecls; }; -// This collects macro expansions/definitions in the main file. -// (Contrast with CollectMainFileMacros in Preamble.cpp, which collects macro -// *definitions* in the preamble region of the main file). -class CollectMainFileMacros : public PPCallbacks { - const SourceManager &SM; - std::vector &MainFileMacroLocs; - - void addLoc(SourceLocation Loc) { - if (!Loc.isMacroID() && isInsideMainFile(Loc, SM)) - MainFileMacroLocs.push_back(Loc); - } - -public: - CollectMainFileMacros(const SourceManager &SM, - std::vector &MainFileMacroLocs) - : SM(SM), MainFileMacroLocs(MainFileMacroLocs) {} - - void MacroDefined(const Token &MacroNameTok, - const MacroDirective *MD) override { - addLoc(MacroNameTok.getLocation()); - } - void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD, - SourceRange Range, const MacroArgs *Args) override { - addLoc(MacroNameTok.getLocation()); - } -}; - // When using a preamble, only preprocessor events outside its bounds are seen. // This is almost what we want: replaying transitive preprocessing wastes time. // However this confuses clang-tidy checks: they don't see any #includes! @@ -362,11 +336,13 @@ // (We can't *just* use the replayed includes, they don't have Resolved path). Clang->getPreprocessor().addPPCallbacks( collectIncludeStructureCallback(Clang->getSourceManager(), &Includes)); - // Collect the macro expansions in the main file. - std::vector MainFileMacroExpLocs; + // Copy over the macros in the preamble region of the main file, and combine + // with non-preamble macros below. + auto MainFileMacros = + Preamble ? Preamble->MainFileMacros : std::vector{}; Clang->getPreprocessor().addPPCallbacks( - std::make_unique(Clang->getSourceManager(), - MainFileMacroExpLocs)); + std::make_unique( + Clang->getSourceManager(), Clang->getLangOpts(), MainFileMacros)); // Copy over the includes from the preamble, then combine with the // non-preamble includes below. @@ -420,7 +396,7 @@ Diags.insert(Diags.end(), D.begin(), D.end()); } return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action), - std::move(Tokens), std::move(MainFileMacroExpLocs), + std::move(Tokens), std::move(MainFileMacros), std::move(ParsedDecls), std::move(Diags), std::move(Includes), std::move(CanonIncludes)); } @@ -460,8 +436,8 @@ return LocalTopLevelDecls; } -llvm::ArrayRef ParsedAST::getMacros() const { - return MacroIdentifierLocs; +llvm::ArrayRef ParsedAST::getMacros() const { + return MainFileMacros; } const std::vector &ParsedAST::getDiagnostics() const { return Diags; } @@ -510,14 +486,13 @@ std::unique_ptr Clang, std::unique_ptr Action, syntax::TokenBuffer Tokens, - std::vector MacroIdentifierLocs, + std::vector MainFileMacros, std::vector LocalTopLevelDecls, std::vector Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes) : Preamble(std::move(Preamble)), Clang(std::move(Clang)), Action(std::move(Action)), Tokens(std::move(Tokens)), - MacroIdentifierLocs(std::move(MacroIdentifierLocs)), - Diags(std::move(Diags)), + MainFileMacros(std::move(MainFileMacros)), Diags(std::move(Diags)), LocalTopLevelDecls(std::move(LocalTopLevelDecls)), Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) { assert(this->Clang); diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h --- a/clang-tools-extra/clangd/Preamble.h +++ b/clang-tools-extra/clangd/Preamble.h @@ -26,6 +26,7 @@ #include "Diagnostics.h" #include "FS.h" #include "Headers.h" +#include "Macro.h" #include "index/CanonicalIncludes.h" #include "clang/Frontend/PrecompiledPreamble.h" #include "clang/Tooling/CompilationDatabase.h" @@ -42,12 +43,6 @@ /// As we must avoid re-parsing the preamble, any information that can only /// be obtained during parsing must be eagerly captured and stored here. struct PreambleData { - PreambleData(PrecompiledPreamble Preamble, std::vector Diags, - IncludeStructure Includes, - std::vector MainFileMacros, - std::unique_ptr StatCache, - CanonicalIncludes CanonIncludes); - tooling::CompileCommand CompileCommand; PrecompiledPreamble Preamble; std::vector Diags; @@ -57,11 +52,17 @@ // Macros defined in the preamble section of the main file. // Users care about headers vs main-file, not preamble vs non-preamble. // These should be treated as main-file entities e.g. for code completion. - std::vector MainFileMacros; + std::vector MainFileMacros; // Cache of FS operations performed when building the preamble. // When reusing a preamble, this cache can be consumed to save IO. std::unique_ptr StatCache; CanonicalIncludes CanonIncludes; + + PreambleData(PrecompiledPreamble Preamble, std::vector Diags, + IncludeStructure Includes, + std::vector MainFileMacros, + std::unique_ptr StatCache, + CanonicalIncludes CanonIncludes); }; using PreambleParsedCallback = diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp --- a/clang-tools-extra/clangd/Preamble.cpp +++ b/clang-tools-extra/clangd/Preamble.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "Preamble.h" +#include "AST.h" #include "Logger.h" #include "Trace.h" #include "clang/Basic/SourceLocation.h" @@ -24,47 +25,14 @@ llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine); } -// This collects macro definitions in the *preamble region* of the main file. -// (Contrast with CollectMainFileMacroExpansions in ParsedAST.cpp, which -// collects macro *expansions* in the rest of the main file. -class CollectMainFileMacros : public PPCallbacks { -public: - explicit CollectMainFileMacros(const SourceManager &SM, - std::vector *Out) - : SM(SM), Out(Out) {} - - void FileChanged(SourceLocation Loc, FileChangeReason, - SrcMgr::CharacteristicKind, FileID Prev) { - InMainFile = SM.isWrittenInMainFile(Loc); - } - - void MacroDefined(const Token &MacroName, const MacroDirective *MD) { - if (InMainFile) - MainFileMacros.insert(MacroName.getIdentifierInfo()->getName()); - } - - void EndOfMainFile() { - for (const auto &Entry : MainFileMacros) - Out->push_back(Entry.getKey()); - llvm::sort(*Out); - } - -private: - const SourceManager &SM; - bool InMainFile = true; - llvm::StringSet<> MainFileMacros; - std::vector *Out; -}; - class CppFilePreambleCallbacks : public PreambleCallbacks { public: CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback) - : File(File), ParsedCallback(ParsedCallback) { - } + : File(File), ParsedCallback(ParsedCallback) {} IncludeStructure takeIncludes() { return std::move(Includes); } - std::vector takeMainFileMacros() { + std::vector takeMainFileMacros() { return std::move(MainFileMacros); } @@ -79,14 +47,18 @@ void BeforeExecute(CompilerInstance &CI) override { CanonIncludes.addSystemHeadersMapping(CI.getLangOpts()); + LangOpts = &CI.getLangOpts(); SourceMgr = &CI.getSourceManager(); } std::unique_ptr createPPCallbacks() override { - assert(SourceMgr && "SourceMgr must be set at this point"); + assert(SourceMgr && LangOpts && + "SourceMgr and LangOpts must be set at this point"); + return std::make_unique( collectIncludeStructureCallback(*SourceMgr, &Includes), - std::make_unique(*SourceMgr, &MainFileMacros)); + std::make_unique(*SourceMgr, *LangOpts, + MainFileMacros)); } CommentHandler *getCommentHandler() override { @@ -99,8 +71,9 @@ PreambleParsedCallback ParsedCallback; IncludeStructure Includes; CanonicalIncludes CanonIncludes; - std::vector MainFileMacros; + std::vector MainFileMacros; std::unique_ptr IWYUHandler = nullptr; + const clang::LangOptions *LangOpts = nullptr; SourceManager *SourceMgr = nullptr; }; @@ -108,7 +81,7 @@ PreambleData::PreambleData(PrecompiledPreamble Preamble, std::vector Diags, IncludeStructure Includes, - std::vector MainFileMacros, + std::vector MainFileMacros, std::unique_ptr StatCache, CanonicalIncludes CanonIncludes) : Preamble(std::move(Preamble)), Diags(std::move(Diags)), diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -38,8 +38,9 @@ TraverseAST(AST.getASTContext()); // Add highlightings for macro expansions as they are not traversed by the // visitor. - for (SourceLocation Loc : AST.getMacros()) - addToken(Loc, HighlightingKind::Macro); + for (const auto &M : AST.getMacros()) + Tokens.push_back({HighlightingKind::Macro, M.R}); + // addToken(Loc, HighlightingKind::Macro); // Initializer lists can give duplicates of tokens, therefore all tokens // must be deduplicated. llvm::sort(Tokens); diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp --- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp +++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp @@ -229,8 +229,8 @@ TEST(ParsedASTTest, CollectsMainFileMacroExpansions) { Annotations TestCase(R"cpp( - #define MACRO_ARGS(X, Y) X Y - // - premable ends, macros inside preamble are not considered in main file. + #define ^MACRO_ARGS(X, Y) X Y + // - preamble ends ^ID(int A); // Macro arguments included. ^MACRO_ARGS(^MACRO_ARGS(^MACRO_EXP(int), A), ^ID(= 2)); @@ -270,12 +270,12 @@ int D = DEF; )cpp"; ParsedAST AST = TU.build(); - const std::vector &MacroExpansionLocations = AST.getMacros(); + auto MacroExpansionLocations = AST.getMacros(); std::vector MacroExpansionPositions; for (const auto &L : MacroExpansionLocations) - MacroExpansionPositions.push_back( - sourceLocToPosition(AST.getSourceManager(), L)); - EXPECT_EQ(MacroExpansionPositions, TestCase.points()); + MacroExpansionPositions.push_back(L.R.start); + EXPECT_THAT(MacroExpansionPositions, + testing::UnorderedElementsAreArray(TestCase.points())); } } // namespace diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp --- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp +++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp @@ -425,8 +425,8 @@ // Tokens that share a source range but have conflicting Kinds are not // highlighted. R"cpp( - #define DEF_MULTIPLE(X) namespace X { class X { int X; }; } - #define DEF_CLASS(T) class T {}; + #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; } + #define $Macro[[DEF_CLASS]](T) class T {}; // Preamble ends. $Macro[[DEF_MULTIPLE]](XYZ); $Macro[[DEF_MULTIPLE]](XYZW); @@ -465,8 +465,8 @@ } )cpp", R"cpp( - #define fail(expr) expr - #define assert(COND) if (!(COND)) { fail("assertion failed" #COND); } + #define $Macro[[fail]](expr) expr + #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); } // Preamble ends. $Primitive[[int]] $Variable[[x]]; $Primitive[[int]] $Variable[[y]];