Skip to content

Commit 04db368

Browse files
ilya-biryukovilya-biryukov
committedJul 21, 2017
[clangd] Replace ASTUnit with manual AST management.
Summary: This refactoring does not aim to introduce any significant changes to the behaviour of clangd to keep the change as simple as possible. Reviewers: klimek, krasimir, bkramer Reviewed By: krasimir Subscribers: malaperle, cfe-commits Tags: #clang-tools-extra Differential Revision: https://reviews.llvm.org/D35406 llvm-svn: 308738
1 parent 04184a6 commit 04db368

File tree

3 files changed

+577
-145
lines changed

3 files changed

+577
-145
lines changed
 

‎clang-tools-extra/clangd/ClangdServer.cpp

+5-8
Original file line numberDiff line numberDiff line change
@@ -205,14 +205,11 @@ ClangdServer::codeComplete(PathRef File, Position Pos,
205205

206206
std::vector<CompletionItem> Result;
207207
auto TaggedFS = FSProvider.getTaggedFileSystem(File);
208-
// It would be nice to use runOnUnitWithoutReparse here, but we can't
209-
// guarantee the correctness of code completion cache here if we don't do the
210-
// reparse.
211-
Units.runOnUnit(File, *OverridenContents, ResourceDir, CDB, PCHs,
212-
TaggedFS.Value, [&](ClangdUnit &Unit) {
213-
Result = Unit.codeComplete(*OverridenContents, Pos,
214-
TaggedFS.Value);
215-
});
208+
Units.runOnUnitWithoutReparse(File, *OverridenContents, ResourceDir, CDB,
209+
PCHs, TaggedFS.Value, [&](ClangdUnit &Unit) {
210+
Result = Unit.codeComplete(
211+
*OverridenContents, Pos, TaggedFS.Value);
212+
});
216213
return make_tagged(std::move(Result), TaggedFS.Tag);
217214
}
218215

‎clang-tools-extra/clangd/ClangdUnit.cpp

+495-134
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,219 @@
99

1010
#include "ClangdUnit.h"
1111

12-
#include "clang/Frontend/ASTUnit.h"
1312
#include "clang/Frontend/CompilerInstance.h"
1413
#include "clang/Frontend/CompilerInvocation.h"
14+
#include "clang/Frontend/FrontendActions.h"
1515
#include "clang/Frontend/Utils.h"
16-
#include "clang/Index/IndexingAction.h"
1716
#include "clang/Index/IndexDataConsumer.h"
17+
#include "clang/Index/IndexingAction.h"
1818
#include "clang/Lex/Lexer.h"
1919
#include "clang/Lex/MacroInfo.h"
2020
#include "clang/Lex/Preprocessor.h"
21+
#include "clang/Lex/PreprocessorOptions.h"
22+
#include "clang/Sema/Sema.h"
23+
#include "clang/Serialization/ASTWriter.h"
2124
#include "clang/Tooling/CompilationDatabase.h"
25+
#include "llvm/ADT/ArrayRef.h"
26+
#include "llvm/ADT/SmallVector.h"
27+
#include "llvm/Support/CrashRecoveryContext.h"
2228
#include "llvm/Support/Format.h"
2329

2430
#include <algorithm>
2531

2632
using namespace clang::clangd;
2733
using namespace clang;
2834

35+
namespace {
36+
37+
class DeclTrackingASTConsumer : public ASTConsumer {
38+
public:
39+
DeclTrackingASTConsumer(std::vector<const Decl *> &TopLevelDecls)
40+
: TopLevelDecls(TopLevelDecls) {}
41+
42+
bool HandleTopLevelDecl(DeclGroupRef DG) override {
43+
for (const Decl *D : DG) {
44+
// ObjCMethodDecl are not actually top-level decls.
45+
if (isa<ObjCMethodDecl>(D))
46+
continue;
47+
48+
TopLevelDecls.push_back(D);
49+
}
50+
return true;
51+
}
52+
53+
private:
54+
std::vector<const Decl *> &TopLevelDecls;
55+
};
56+
57+
class ClangdFrontendAction : public SyntaxOnlyAction {
58+
public:
59+
std::vector<const Decl *> takeTopLevelDecls() {
60+
return std::move(TopLevelDecls);
61+
}
62+
63+
protected:
64+
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
65+
StringRef InFile) override {
66+
return llvm::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
67+
}
68+
69+
private:
70+
std::vector<const Decl *> TopLevelDecls;
71+
};
72+
73+
class ClangdUnitPreambleCallbacks : public PreambleCallbacks {
74+
public:
75+
std::vector<serialization::DeclID> takeTopLevelDeclIDs() {
76+
return std::move(TopLevelDeclIDs);
77+
}
78+
79+
void AfterPCHEmitted(ASTWriter &Writer) override {
80+
TopLevelDeclIDs.reserve(TopLevelDecls.size());
81+
for (Decl *D : TopLevelDecls) {
82+
// Invalid top-level decls may not have been serialized.
83+
if (D->isInvalidDecl())
84+
continue;
85+
TopLevelDeclIDs.push_back(Writer.getDeclID(D));
86+
}
87+
}
88+
89+
void HandleTopLevelDecl(DeclGroupRef DG) override {
90+
for (Decl *D : DG) {
91+
if (isa<ObjCMethodDecl>(D))
92+
continue;
93+
TopLevelDecls.push_back(D);
94+
}
95+
}
96+
97+
private:
98+
std::vector<Decl *> TopLevelDecls;
99+
std::vector<serialization::DeclID> TopLevelDeclIDs;
100+
};
101+
102+
/// Convert from clang diagnostic level to LSP severity.
103+
static int getSeverity(DiagnosticsEngine::Level L) {
104+
switch (L) {
105+
case DiagnosticsEngine::Remark:
106+
return 4;
107+
case DiagnosticsEngine::Note:
108+
return 3;
109+
case DiagnosticsEngine::Warning:
110+
return 2;
111+
case DiagnosticsEngine::Fatal:
112+
case DiagnosticsEngine::Error:
113+
return 1;
114+
case DiagnosticsEngine::Ignored:
115+
return 0;
116+
}
117+
llvm_unreachable("Unknown diagnostic level!");
118+
}
119+
120+
llvm::Optional<DiagWithFixIts> toClangdDiag(StoredDiagnostic D) {
121+
auto Location = D.getLocation();
122+
if (!Location.isValid() || !Location.getManager().isInMainFile(Location))
123+
return llvm::None;
124+
125+
Position P;
126+
P.line = Location.getSpellingLineNumber() - 1;
127+
P.character = Location.getSpellingColumnNumber();
128+
Range R = {P, P};
129+
clangd::Diagnostic Diag = {R, getSeverity(D.getLevel()), D.getMessage()};
130+
131+
llvm::SmallVector<tooling::Replacement, 1> FixItsForDiagnostic;
132+
for (const FixItHint &Fix : D.getFixIts()) {
133+
FixItsForDiagnostic.push_back(clang::tooling::Replacement(
134+
Location.getManager(), Fix.RemoveRange, Fix.CodeToInsert));
135+
}
136+
return DiagWithFixIts{Diag, std::move(FixItsForDiagnostic)};
137+
}
138+
139+
class StoreDiagsConsumer : public DiagnosticConsumer {
140+
public:
141+
StoreDiagsConsumer(std::vector<DiagWithFixIts> &Output) : Output(Output) {}
142+
143+
void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
144+
const clang::Diagnostic &Info) override {
145+
DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
146+
147+
if (auto convertedDiag = toClangdDiag(StoredDiagnostic(DiagLevel, Info)))
148+
Output.push_back(std::move(*convertedDiag));
149+
}
150+
151+
private:
152+
std::vector<DiagWithFixIts> &Output;
153+
};
154+
155+
class EmptyDiagsConsumer : public DiagnosticConsumer {
156+
public:
157+
void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
158+
const clang::Diagnostic &Info) override {}
159+
};
160+
161+
std::unique_ptr<CompilerInvocation>
162+
createCompilerInvocation(ArrayRef<const char *> ArgList,
163+
IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
164+
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
165+
auto CI = createInvocationFromCommandLine(ArgList, std::move(Diags),
166+
std::move(VFS));
167+
// We rely on CompilerInstance to manage the resource (i.e. free them on
168+
// EndSourceFile), but that won't happen if DisableFree is set to true.
169+
// Since createInvocationFromCommandLine sets it to true, we have to override
170+
// it.
171+
CI->getFrontendOpts().DisableFree = false;
172+
return CI;
173+
}
174+
175+
/// Creates a CompilerInstance from \p CI, with main buffer overriden to \p
176+
/// Buffer and arguments to read the PCH from \p Preamble, if \p Preamble is not
177+
/// null. Note that vfs::FileSystem inside returned instance may differ from \p
178+
/// VFS if additional file remapping were set in command-line arguments.
179+
/// On some errors, returns null. When non-null value is returned, it's expected
180+
/// to be consumed by the FrontendAction as it will have a pointer to the \p
181+
/// Buffer that will only be deleted if BeginSourceFile is called.
182+
std::unique_ptr<CompilerInstance>
183+
prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI,
184+
const PrecompiledPreamble *Preamble,
185+
std::unique_ptr<llvm::MemoryBuffer> Buffer,
186+
std::shared_ptr<PCHContainerOperations> PCHs,
187+
IntrusiveRefCntPtr<vfs::FileSystem> VFS,
188+
DiagnosticConsumer &DiagsClient) {
189+
assert(VFS && "VFS is null");
190+
assert(!CI->getPreprocessorOpts().RetainRemappedFileBuffers &&
191+
"Setting RetainRemappedFileBuffers to true will cause a memory leak "
192+
"of ContentsBuffer");
193+
194+
// NOTE: we use Buffer.get() when adding remapped files, so we have to make
195+
// sure it will be released if no error is emitted.
196+
if (Preamble) {
197+
Preamble->AddImplicitPreamble(*CI, Buffer.get());
198+
} else {
199+
CI->getPreprocessorOpts().addRemappedFile(
200+
CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get());
201+
}
202+
203+
auto Clang = llvm::make_unique<CompilerInstance>(PCHs);
204+
Clang->setInvocation(std::move(CI));
205+
Clang->createDiagnostics(&DiagsClient, false);
206+
207+
if (auto VFSWithRemapping = createVFSFromCompilerInvocation(
208+
Clang->getInvocation(), Clang->getDiagnostics(), VFS))
209+
VFS = VFSWithRemapping;
210+
Clang->setVirtualFileSystem(VFS);
211+
212+
Clang->setTarget(TargetInfo::CreateTargetInfo(
213+
Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
214+
if (!Clang->hasTarget())
215+
return nullptr;
216+
217+
// RemappedFileBuffers will handle the lifetime of the Buffer pointer,
218+
// release it.
219+
Buffer.release();
220+
return Clang;
221+
}
222+
223+
} // namespace
224+
29225
ClangdUnit::ClangdUnit(PathRef FileName, StringRef Contents,
30226
StringRef ResourceDir,
31227
std::shared_ptr<PCHContainerOperations> PCHs,
@@ -39,44 +235,54 @@ ClangdUnit::ClangdUnit(PathRef FileName, StringRef Contents,
39235
Commands.front().CommandLine.push_back("-resource-dir=" +
40236
std::string(ResourceDir));
41237

42-
IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
43-
CompilerInstance::createDiagnostics(new DiagnosticOptions);
44-
45-
std::vector<const char *> ArgStrs;
46-
for (const auto &S : Commands.front().CommandLine)
47-
ArgStrs.push_back(S.c_str());
48-
49-
ASTUnit::RemappedFile RemappedSource(
50-
FileName,
51-
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName).release());
52-
53-
auto ArgP = &*ArgStrs.begin();
54-
Unit = std::unique_ptr<ASTUnit>(ASTUnit::LoadFromCommandLine(
55-
ArgP, ArgP + ArgStrs.size(), PCHs, Diags, ResourceDir,
56-
/*OnlyLocalDecls=*/false, /*CaptureDiagnostics=*/true, RemappedSource,
57-
/*RemappedFilesKeepOriginalName=*/true,
58-
/*PrecompilePreambleAfterNParses=*/1, /*TUKind=*/TU_Prefix,
59-
/*CacheCodeCompletionResults=*/true,
60-
/*IncludeBriefCommentsInCodeCompletion=*/true,
61-
/*AllowPCHWithCompilerErrors=*/true,
62-
/*SkipFunctionBodies=*/false,
63-
/*SingleFileParse=*/false,
64-
/*UserFilesAreVolatile=*/false, /*ForSerialization=*/false,
65-
/*ModuleFormat=*/llvm::None,
66-
/*ErrAST=*/nullptr, VFS));
67-
assert(Unit && "Unit wasn't created");
238+
Command = std::move(Commands.front());
239+
reparse(Contents, VFS);
68240
}
69241

70242
void ClangdUnit::reparse(StringRef Contents,
71243
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
72-
// Do a reparse if this wasn't the first parse.
73-
// FIXME: This might have the wrong working directory if it changed in the
74-
// meantime.
75-
ASTUnit::RemappedFile RemappedSource(
76-
FileName,
77-
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName).release());
78-
79-
Unit->Reparse(PCHs, RemappedSource, VFS);
244+
std::vector<const char *> ArgStrs;
245+
for (const auto &S : Command.CommandLine)
246+
ArgStrs.push_back(S.c_str());
247+
248+
std::unique_ptr<CompilerInvocation> CI;
249+
{
250+
// FIXME(ibiryukov): store diagnostics from CommandLine when we start
251+
// reporting them.
252+
EmptyDiagsConsumer CommandLineDiagsConsumer;
253+
IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
254+
CompilerInstance::createDiagnostics(new DiagnosticOptions,
255+
&CommandLineDiagsConsumer, false);
256+
CI = createCompilerInvocation(ArgStrs, CommandLineDiagsEngine, VFS);
257+
}
258+
assert(CI && "Couldn't create CompilerInvocation");
259+
260+
std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
261+
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName);
262+
263+
// Rebuild the preamble if it is missing or can not be reused.
264+
auto Bounds =
265+
ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
266+
if (!Preamble || !Preamble->Preamble.CanReuse(*CI, ContentsBuffer.get(),
267+
Bounds, VFS.get())) {
268+
std::vector<DiagWithFixIts> PreambleDiags;
269+
StoreDiagsConsumer PreambleDiagnosticsConsumer(/*ref*/ PreambleDiags);
270+
IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
271+
CompilerInstance::createDiagnostics(
272+
&CI->getDiagnosticOpts(), &PreambleDiagnosticsConsumer, false);
273+
ClangdUnitPreambleCallbacks SerializedDeclsCollector;
274+
auto BuiltPreamble = PrecompiledPreamble::Build(
275+
*CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, VFS, PCHs,
276+
SerializedDeclsCollector);
277+
if (BuiltPreamble)
278+
Preamble = PreambleData(std::move(*BuiltPreamble),
279+
SerializedDeclsCollector.takeTopLevelDeclIDs(),
280+
std::move(PreambleDiags));
281+
}
282+
Unit = ParsedAST::Build(
283+
std::move(CI), Preamble ? &Preamble->Preamble : nullptr,
284+
Preamble ? llvm::makeArrayRef(Preamble->TopLevelDeclIDs) : llvm::None,
285+
std::move(ContentsBuffer), PCHs, VFS);
80286
}
81287

82288
namespace {
@@ -188,97 +394,164 @@ class CompletionItemsCollector : public CodeCompleteConsumer {
188394
std::vector<CompletionItem>
189395
ClangdUnit::codeComplete(StringRef Contents, Position Pos,
190396
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
191-
CodeCompleteOptions CCO;
192-
CCO.IncludeBriefComments = 1;
193-
// This is where code completion stores dirty buffers. Need to free after
194-
// completion.
195-
SmallVector<const llvm::MemoryBuffer *, 4> OwnedBuffers;
196-
SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
197-
IntrusiveRefCntPtr<DiagnosticsEngine> DiagEngine(
198-
new DiagnosticsEngine(new DiagnosticIDs, new DiagnosticOptions));
397+
std::vector<const char *> ArgStrs;
398+
for (const auto &S : Command.CommandLine)
399+
ArgStrs.push_back(S.c_str());
400+
401+
std::unique_ptr<CompilerInvocation> CI;
402+
EmptyDiagsConsumer DummyDiagsConsumer;
403+
{
404+
IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine =
405+
CompilerInstance::createDiagnostics(new DiagnosticOptions,
406+
&DummyDiagsConsumer, false);
407+
CI = createCompilerInvocation(ArgStrs, CommandLineDiagsEngine, VFS);
408+
}
409+
assert(CI && "Couldn't create CompilerInvocation");
410+
411+
std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
412+
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName);
413+
414+
// Attempt to reuse the PCH from precompiled preamble, if it was built.
415+
const PrecompiledPreamble *PreambleForCompletion = nullptr;
416+
if (Preamble) {
417+
auto Bounds =
418+
ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
419+
if (Preamble->Preamble.CanReuse(*CI, ContentsBuffer.get(), Bounds,
420+
VFS.get()))
421+
PreambleForCompletion = &Preamble->Preamble;
422+
}
423+
424+
auto Clang = prepareCompilerInstance(std::move(CI), PreambleForCompletion,
425+
std::move(ContentsBuffer), PCHs, VFS,
426+
DummyDiagsConsumer);
427+
auto &DiagOpts = Clang->getDiagnosticOpts();
428+
DiagOpts.IgnoreWarnings = true;
429+
430+
auto &FrontendOpts = Clang->getFrontendOpts();
431+
FrontendOpts.SkipFunctionBodies = true;
432+
433+
FrontendOpts.CodeCompleteOpts.IncludeGlobals = true;
434+
// we don't handle code patterns properly yet, disable them.
435+
FrontendOpts.CodeCompleteOpts.IncludeCodePatterns = false;
436+
FrontendOpts.CodeCompleteOpts.IncludeMacros = true;
437+
FrontendOpts.CodeCompleteOpts.IncludeBriefComments = true;
438+
439+
FrontendOpts.CodeCompletionAt.FileName = FileName;
440+
FrontendOpts.CodeCompletionAt.Line = Pos.line + 1;
441+
FrontendOpts.CodeCompletionAt.Column = Pos.character + 1;
442+
199443
std::vector<CompletionItem> Items;
200-
CompletionItemsCollector Collector(&Items, CCO);
201-
202-
ASTUnit::RemappedFile RemappedSource(
203-
FileName,
204-
llvm::MemoryBuffer::getMemBufferCopy(Contents, FileName).release());
205-
206-
IntrusiveRefCntPtr<FileManager> FileMgr(
207-
new FileManager(Unit->getFileSystemOpts(), VFS));
208-
IntrusiveRefCntPtr<SourceManager> SourceMgr(
209-
new SourceManager(*DiagEngine, *FileMgr));
210-
// CodeComplete seems to require fresh LangOptions.
211-
LangOptions LangOpts = Unit->getLangOpts();
212-
// The language server protocol uses zero-based line and column numbers.
213-
// The clang code completion uses one-based numbers.
214-
Unit->CodeComplete(FileName, Pos.line + 1, Pos.character + 1, RemappedSource,
215-
CCO.IncludeMacros, CCO.IncludeCodePatterns,
216-
CCO.IncludeBriefComments, Collector, PCHs, *DiagEngine,
217-
LangOpts, *SourceMgr, *FileMgr, StoredDiagnostics,
218-
OwnedBuffers);
219-
for (const llvm::MemoryBuffer *Buffer : OwnedBuffers)
220-
delete Buffer;
221-
return Items;
222-
}
444+
Clang->setCodeCompletionConsumer(
445+
new CompletionItemsCollector(&Items, FrontendOpts.CodeCompleteOpts));
223446

224-
namespace {
225-
/// Convert from clang diagnostic level to LSP severity.
226-
static int getSeverity(DiagnosticsEngine::Level L) {
227-
switch (L) {
228-
case DiagnosticsEngine::Remark:
229-
return 4;
230-
case DiagnosticsEngine::Note:
231-
return 3;
232-
case DiagnosticsEngine::Warning:
233-
return 2;
234-
case DiagnosticsEngine::Fatal:
235-
case DiagnosticsEngine::Error:
236-
return 1;
237-
case DiagnosticsEngine::Ignored:
238-
return 0;
447+
SyntaxOnlyAction Action;
448+
if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) {
449+
// FIXME(ibiryukov): log errors
450+
return Items;
239451
}
240-
llvm_unreachable("Unknown diagnostic level!");
452+
if (!Action.Execute()) {
453+
// FIXME(ibiryukov): log errors
454+
}
455+
Action.EndSourceFile();
456+
457+
return Items;
241458
}
242-
} // namespace
243459

244460
std::vector<DiagWithFixIts> ClangdUnit::getLocalDiagnostics() const {
461+
if (!Unit)
462+
return {}; // Parsing failed.
463+
245464
std::vector<DiagWithFixIts> Result;
246-
for (ASTUnit::stored_diag_iterator D = Unit->stored_diag_begin(),
247-
DEnd = Unit->stored_diag_end();
248-
D != DEnd; ++D) {
249-
if (!D->getLocation().isValid() ||
250-
!D->getLocation().getManager().isInMainFile(D->getLocation()))
251-
continue;
252-
Position P;
253-
P.line = D->getLocation().getSpellingLineNumber() - 1;
254-
P.character = D->getLocation().getSpellingColumnNumber();
255-
Range R = {P, P};
256-
clangd::Diagnostic Diag = {R, getSeverity(D->getLevel()), D->getMessage()};
257-
258-
llvm::SmallVector<tooling::Replacement, 1> FixItsForDiagnostic;
259-
for (const FixItHint &Fix : D->getFixIts()) {
260-
FixItsForDiagnostic.push_back(clang::tooling::Replacement(
261-
Unit->getSourceManager(), Fix.RemoveRange, Fix.CodeToInsert));
262-
}
263-
Result.push_back({Diag, std::move(FixItsForDiagnostic)});
264-
}
465+
auto PreambleDiagsSize = Preamble ? Preamble->Diags.size() : 0;
466+
const auto &Diags = Unit->getDiagnostics();
467+
Result.reserve(PreambleDiagsSize + Diags.size());
468+
469+
if (Preamble)
470+
Result.insert(Result.end(), Preamble->Diags.begin(), Preamble->Diags.end());
471+
Result.insert(Result.end(), Diags.begin(), Diags.end());
265472
return Result;
266473
}
267474

268475
void ClangdUnit::dumpAST(llvm::raw_ostream &OS) const {
476+
if (!Unit) {
477+
OS << "<no-ast-in-clang>";
478+
return; // Parsing failed.
479+
}
269480
Unit->getASTContext().getTranslationUnitDecl()->dump(OS, true);
270481
}
271482

483+
llvm::Optional<ClangdUnit::ParsedAST>
484+
ClangdUnit::ParsedAST::Build(std::unique_ptr<clang::CompilerInvocation> CI,
485+
const PrecompiledPreamble *Preamble,
486+
ArrayRef<serialization::DeclID> PreambleDeclIDs,
487+
std::unique_ptr<llvm::MemoryBuffer> Buffer,
488+
std::shared_ptr<PCHContainerOperations> PCHs,
489+
IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
490+
491+
std::vector<DiagWithFixIts> ASTDiags;
492+
StoreDiagsConsumer UnitDiagsConsumer(/*ref*/ ASTDiags);
493+
494+
auto Clang =
495+
prepareCompilerInstance(std::move(CI), Preamble, std::move(Buffer), PCHs,
496+
VFS, /*ref*/ UnitDiagsConsumer);
497+
498+
// Recover resources if we crash before exiting this method.
499+
llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
500+
Clang.get());
501+
502+
auto Action = llvm::make_unique<ClangdFrontendAction>();
503+
if (!Action->BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) {
504+
// FIXME(ibiryukov): log error
505+
return llvm::None;
506+
}
507+
if (!Action->Execute()) {
508+
// FIXME(ibiryukov): log error
509+
}
510+
511+
// UnitDiagsConsumer is local, we can not store it in CompilerInstance that
512+
// has a longer lifetime.
513+
Clang->getDiagnostics().setClient(new EmptyDiagsConsumer);
514+
515+
std::vector<const Decl *> ParsedDecls = Action->takeTopLevelDecls();
516+
std::vector<serialization::DeclID> PendingDecls;
517+
if (Preamble) {
518+
PendingDecls.reserve(PreambleDeclIDs.size());
519+
PendingDecls.insert(PendingDecls.begin(), PreambleDeclIDs.begin(),
520+
PreambleDeclIDs.end());
521+
}
522+
523+
return ParsedAST(std::move(Clang), std::move(Action), std::move(ParsedDecls),
524+
std::move(PendingDecls), std::move(ASTDiags));
525+
}
526+
272527
namespace {
528+
529+
SourceLocation getMacroArgExpandedLocation(const SourceManager &Mgr,
530+
const FileEntry *FE,
531+
unsigned Offset) {
532+
SourceLocation FileLoc = Mgr.translateFileLineCol(FE, 1, 1);
533+
return Mgr.getMacroArgExpandedLocation(FileLoc.getLocWithOffset(Offset));
534+
}
535+
536+
SourceLocation getMacroArgExpandedLocation(const SourceManager &Mgr,
537+
const FileEntry *FE, Position Pos) {
538+
SourceLocation InputLoc =
539+
Mgr.translateFileLineCol(FE, Pos.line + 1, Pos.character + 1);
540+
return Mgr.getMacroArgExpandedLocation(InputLoc);
541+
}
542+
273543
/// Finds declarations locations that a given source location refers to.
274544
class DeclarationLocationsFinder : public index::IndexDataConsumer {
275545
std::vector<Location> DeclarationLocations;
276546
const SourceLocation &SearchedLocation;
277-
ASTUnit &Unit;
547+
const ASTContext &AST;
548+
Preprocessor &PP;
549+
278550
public:
279551
DeclarationLocationsFinder(raw_ostream &OS,
280-
const SourceLocation &SearchedLocation, ASTUnit &Unit) :
281-
SearchedLocation(SearchedLocation), Unit(Unit) {}
552+
const SourceLocation &SearchedLocation,
553+
ASTContext &AST, Preprocessor &PP)
554+
: SearchedLocation(SearchedLocation), AST(AST), PP(PP) {}
282555

283556
std::vector<Location> takeLocations() {
284557
// Don't keep the same location multiple times.
@@ -302,17 +575,17 @@ class DeclarationLocationsFinder : public index::IndexDataConsumer {
302575

303576
private:
304577
bool isSearchedLocation(FileID FID, unsigned Offset) const {
305-
const SourceManager &SourceMgr = Unit.getSourceManager();
306-
return SourceMgr.getFileOffset(SearchedLocation) == Offset
307-
&& SourceMgr.getFileID(SearchedLocation) == FID;
578+
const SourceManager &SourceMgr = AST.getSourceManager();
579+
return SourceMgr.getFileOffset(SearchedLocation) == Offset &&
580+
SourceMgr.getFileID(SearchedLocation) == FID;
308581
}
309582

310-
void addDeclarationLocation(const SourceRange& ValSourceRange) {
311-
const SourceManager& SourceMgr = Unit.getSourceManager();
312-
const LangOptions& LangOpts = Unit.getLangOpts();
583+
void addDeclarationLocation(const SourceRange &ValSourceRange) {
584+
const SourceManager &SourceMgr = AST.getSourceManager();
585+
const LangOptions &LangOpts = AST.getLangOpts();
313586
SourceLocation LocStart = ValSourceRange.getBegin();
314587
SourceLocation LocEnd = Lexer::getLocForEndOfToken(ValSourceRange.getEnd(),
315-
0, SourceMgr, LangOpts);
588+
0, SourceMgr, LangOpts);
316589
Position Begin;
317590
Begin.line = SourceMgr.getSpellingLineNumber(LocStart) - 1;
318591
Begin.character = SourceMgr.getSpellingColumnNumber(LocStart) - 1;
@@ -330,24 +603,24 @@ class DeclarationLocationsFinder : public index::IndexDataConsumer {
330603
void finish() override {
331604
// Also handle possible macro at the searched location.
332605
Token Result;
333-
if (!Lexer::getRawToken(SearchedLocation, Result, Unit.getSourceManager(),
334-
Unit.getASTContext().getLangOpts(), false)) {
606+
if (!Lexer::getRawToken(SearchedLocation, Result, AST.getSourceManager(),
607+
AST.getLangOpts(), false)) {
335608
if (Result.is(tok::raw_identifier)) {
336-
Unit.getPreprocessor().LookUpIdentifierInfo(Result);
609+
PP.LookUpIdentifierInfo(Result);
337610
}
338-
IdentifierInfo* IdentifierInfo = Result.getIdentifierInfo();
611+
IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo();
339612
if (IdentifierInfo && IdentifierInfo->hadMacroDefinition()) {
340613
std::pair<FileID, unsigned int> DecLoc =
341-
Unit.getSourceManager().getDecomposedExpansionLoc(SearchedLocation);
614+
AST.getSourceManager().getDecomposedExpansionLoc(SearchedLocation);
342615
// Get the definition just before the searched location so that a macro
343616
// referenced in a '#undef MACRO' can still be found.
344-
SourceLocation BeforeSearchedLocation = Unit.getLocation(
345-
Unit.getSourceManager().getFileEntryForID(DecLoc.first),
617+
SourceLocation BeforeSearchedLocation = getMacroArgExpandedLocation(
618+
AST.getSourceManager(),
619+
AST.getSourceManager().getFileEntryForID(DecLoc.first),
346620
DecLoc.second - 1);
347621
MacroDefinition MacroDef =
348-
Unit.getPreprocessor().getMacroDefinitionAtLoc(IdentifierInfo,
349-
BeforeSearchedLocation);
350-
MacroInfo* MacroInf = MacroDef.getMacroInfo();
622+
PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation);
623+
MacroInfo *MacroInf = MacroDef.getMacroInfo();
351624
if (MacroInf) {
352625
addDeclarationLocation(
353626
SourceRange(MacroInf->getDefinitionLoc(),
@@ -360,30 +633,41 @@ class DeclarationLocationsFinder : public index::IndexDataConsumer {
360633
} // namespace
361634

362635
std::vector<Location> ClangdUnit::findDefinitions(Position Pos) {
363-
const FileEntry *FE = Unit->getFileManager().getFile(Unit->getMainFileName());
636+
if (!Unit)
637+
return {}; // Parsing failed.
638+
639+
const SourceManager &SourceMgr = Unit->getASTContext().getSourceManager();
640+
const FileEntry *FE = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID());
364641
if (!FE)
365642
return {};
366643

367644
SourceLocation SourceLocationBeg = getBeginningOfIdentifier(Pos, FE);
368645

369646
auto DeclLocationsFinder = std::make_shared<DeclarationLocationsFinder>(
370-
llvm::errs(), SourceLocationBeg, *Unit);
647+
llvm::errs(), SourceLocationBeg, Unit->getASTContext(),
648+
Unit->getPreprocessor());
371649
index::IndexingOptions IndexOpts;
372650
IndexOpts.SystemSymbolFilter =
373651
index::IndexingOptions::SystemSymbolFilterKind::All;
374652
IndexOpts.IndexFunctionLocals = true;
375-
index::indexASTUnit(*Unit, DeclLocationsFinder, IndexOpts);
653+
654+
indexTopLevelDecls(Unit->getASTContext(), Unit->getTopLevelDecls(),
655+
DeclLocationsFinder, IndexOpts);
376656

377657
return DeclLocationsFinder->takeLocations();
378658
}
379659

380660
SourceLocation ClangdUnit::getBeginningOfIdentifier(const Position &Pos,
381-
const FileEntry *FE) const {
661+
const FileEntry *FE) const {
662+
382663
// The language server protocol uses zero-based line and column numbers.
383664
// Clang uses one-based numbers.
384-
SourceLocation InputLocation = Unit->getLocation(FE, Pos.line + 1,
385-
Pos.character + 1);
386665

666+
const ASTContext &AST = Unit->getASTContext();
667+
const SourceManager &SourceMgr = AST.getSourceManager();
668+
669+
SourceLocation InputLocation =
670+
getMacroArgExpandedLocation(SourceMgr, FE, Pos);
387671
if (Pos.character == 0) {
388672
return InputLocation;
389673
}
@@ -396,20 +680,97 @@ SourceLocation ClangdUnit::getBeginningOfIdentifier(const Position &Pos,
396680
// token. If so, Take the beginning of this token.
397681
// (It should be the same identifier because you can't have two adjacent
398682
// identifiers without another token in between.)
399-
SourceLocation PeekBeforeLocation = Unit->getLocation(FE, Pos.line + 1,
400-
Pos.character);
401-
const SourceManager &SourceMgr = Unit->getSourceManager();
683+
SourceLocation PeekBeforeLocation = getMacroArgExpandedLocation(
684+
SourceMgr, FE, Position{Pos.line, Pos.character - 1});
402685
Token Result;
403686
if (Lexer::getRawToken(PeekBeforeLocation, Result, SourceMgr,
404-
Unit->getASTContext().getLangOpts(), false)) {
687+
AST.getLangOpts(), false)) {
405688
// getRawToken failed, just use InputLocation.
406689
return InputLocation;
407690
}
408691

409692
if (Result.is(tok::raw_identifier)) {
410693
return Lexer::GetBeginningOfToken(PeekBeforeLocation, SourceMgr,
411-
Unit->getASTContext().getLangOpts());
694+
Unit->getASTContext().getLangOpts());
412695
}
413696

414697
return InputLocation;
415698
}
699+
700+
void ClangdUnit::ParsedAST::ensurePreambleDeclsDeserialized() {
701+
if (PendingTopLevelDecls.empty())
702+
return;
703+
704+
std::vector<const Decl *> Resolved;
705+
Resolved.reserve(PendingTopLevelDecls.size());
706+
707+
ExternalASTSource &Source = *getASTContext().getExternalSource();
708+
for (serialization::DeclID TopLevelDecl : PendingTopLevelDecls) {
709+
// Resolve the declaration ID to an actual declaration, possibly
710+
// deserializing the declaration in the process.
711+
if (Decl *D = Source.GetExternalDecl(TopLevelDecl))
712+
Resolved.push_back(D);
713+
}
714+
715+
TopLevelDecls.reserve(TopLevelDecls.size() + PendingTopLevelDecls.size());
716+
TopLevelDecls.insert(TopLevelDecls.begin(), Resolved.begin(), Resolved.end());
717+
718+
PendingTopLevelDecls.clear();
719+
}
720+
721+
ClangdUnit::ParsedAST::ParsedAST(ParsedAST &&Other) = default;
722+
723+
ClangdUnit::ParsedAST &ClangdUnit::ParsedAST::
724+
operator=(ParsedAST &&Other) = default;
725+
726+
ClangdUnit::ParsedAST::~ParsedAST() {
727+
if (Action) {
728+
Action->EndSourceFile();
729+
}
730+
}
731+
732+
ASTContext &ClangdUnit::ParsedAST::getASTContext() {
733+
return Clang->getASTContext();
734+
}
735+
736+
const ASTContext &ClangdUnit::ParsedAST::getASTContext() const {
737+
return Clang->getASTContext();
738+
}
739+
740+
Preprocessor &ClangdUnit::ParsedAST::getPreprocessor() {
741+
return Clang->getPreprocessor();
742+
}
743+
744+
const Preprocessor &ClangdUnit::ParsedAST::getPreprocessor() const {
745+
return Clang->getPreprocessor();
746+
}
747+
748+
ArrayRef<const Decl *> ClangdUnit::ParsedAST::getTopLevelDecls() {
749+
ensurePreambleDeclsDeserialized();
750+
return TopLevelDecls;
751+
}
752+
753+
const std::vector<DiagWithFixIts> &
754+
ClangdUnit::ParsedAST::getDiagnostics() const {
755+
return Diags;
756+
}
757+
758+
ClangdUnit::ParsedAST::ParsedAST(
759+
std::unique_ptr<CompilerInstance> Clang,
760+
std::unique_ptr<FrontendAction> Action,
761+
std::vector<const Decl *> TopLevelDecls,
762+
std::vector<serialization::DeclID> PendingTopLevelDecls,
763+
std::vector<DiagWithFixIts> Diags)
764+
: Clang(std::move(Clang)), Action(std::move(Action)),
765+
Diags(std::move(Diags)), TopLevelDecls(std::move(TopLevelDecls)),
766+
PendingTopLevelDecls(std::move(PendingTopLevelDecls)) {
767+
assert(this->Clang);
768+
assert(this->Action);
769+
}
770+
771+
ClangdUnit::PreambleData::PreambleData(
772+
PrecompiledPreamble Preamble,
773+
std::vector<serialization::DeclID> TopLevelDeclIDs,
774+
std::vector<DiagWithFixIts> Diags)
775+
: Preamble(std::move(Preamble)),
776+
TopLevelDeclIDs(std::move(TopLevelDeclIDs)), Diags(std::move(Diags)) {}

‎clang-tools-extra/clangd/ClangdUnit.h

+77-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
#include "Path.h"
1414
#include "Protocol.h"
1515
#include "clang/Frontend/ASTUnit.h"
16+
#include "clang/Frontend/PrecompiledPreamble.h"
17+
#include "clang/Serialization/ASTBitCodes.h"
18+
#include "clang/Tooling/CompilationDatabase.h"
1619
#include "clang/Tooling/Core/Replacement.h"
1720
#include <memory>
1821

@@ -70,11 +73,82 @@ class ClangdUnit {
7073
void dumpAST(llvm::raw_ostream &OS) const;
7174

7275
private:
76+
/// Stores and provides access to parsed AST.
77+
class ParsedAST {
78+
public:
79+
/// Attempts to run Clang and store parsed AST. If \p Preamble is non-null
80+
/// it is reused during parsing.
81+
static llvm::Optional<ParsedAST>
82+
Build(std::unique_ptr<clang::CompilerInvocation> CI,
83+
const PrecompiledPreamble *Preamble,
84+
ArrayRef<serialization::DeclID> PreambleDeclIDs,
85+
std::unique_ptr<llvm::MemoryBuffer> Buffer,
86+
std::shared_ptr<PCHContainerOperations> PCHs,
87+
IntrusiveRefCntPtr<vfs::FileSystem> VFS);
88+
89+
ParsedAST(ParsedAST &&Other);
90+
ParsedAST &operator=(ParsedAST &&Other);
91+
92+
~ParsedAST();
93+
94+
ASTContext &getASTContext();
95+
const ASTContext &getASTContext() const;
96+
97+
Preprocessor &getPreprocessor();
98+
const Preprocessor &getPreprocessor() const;
99+
100+
/// This function returns all top-level decls, including those that come
101+
/// from Preamble. Decls, coming from Preamble, have to be deserialized, so
102+
/// this call might be expensive.
103+
ArrayRef<const Decl *> getTopLevelDecls();
104+
105+
const std::vector<DiagWithFixIts> &getDiagnostics() const;
106+
107+
private:
108+
ParsedAST(std::unique_ptr<CompilerInstance> Clang,
109+
std::unique_ptr<FrontendAction> Action,
110+
std::vector<const Decl *> TopLevelDecls,
111+
std::vector<serialization::DeclID> PendingTopLevelDecls,
112+
std::vector<DiagWithFixIts> Diags);
113+
114+
private:
115+
void ensurePreambleDeclsDeserialized();
116+
117+
// We store an "incomplete" FrontendAction (i.e. no EndSourceFile was called
118+
// on it) and CompilerInstance used to run it. That way we don't have to do
119+
// complex memory management of all Clang structures on our own. (They are
120+
// stored in CompilerInstance and cleaned up by
121+
// FrontendAction.EndSourceFile).
122+
std::unique_ptr<CompilerInstance> Clang;
123+
std::unique_ptr<FrontendAction> Action;
124+
125+
// Data, stored after parsing.
126+
std::vector<DiagWithFixIts> Diags;
127+
std::vector<const Decl *> TopLevelDecls;
128+
std::vector<serialization::DeclID> PendingTopLevelDecls;
129+
};
130+
131+
// Store Preamble and all associated data
132+
struct PreambleData {
133+
PreambleData(PrecompiledPreamble Preamble,
134+
std::vector<serialization::DeclID> TopLevelDeclIDs,
135+
std::vector<DiagWithFixIts> Diags);
136+
137+
PrecompiledPreamble Preamble;
138+
std::vector<serialization::DeclID> TopLevelDeclIDs;
139+
std::vector<DiagWithFixIts> Diags;
140+
};
141+
142+
SourceLocation getBeginningOfIdentifier(const Position &Pos,
143+
const FileEntry *FE) const;
144+
73145
Path FileName;
74-
std::unique_ptr<ASTUnit> Unit;
75-
std::shared_ptr<PCHContainerOperations> PCHs;
146+
tooling::CompileCommand Command;
76147

77-
SourceLocation getBeginningOfIdentifier(const Position& Pos, const FileEntry* FE) const;
148+
llvm::Optional<PreambleData> Preamble;
149+
llvm::Optional<ParsedAST> Unit;
150+
151+
std::shared_ptr<PCHContainerOperations> PCHs;
78152
};
79153

80154
} // namespace clangd

0 commit comments

Comments
 (0)
Please sign in to comment.