diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -131,7 +131,7 @@ // If the file is open in user's editor, make sure the version we // saw and current version are compatible as this is the text that // will be replaced by editors. - if (!It.second.canApplyTo(Draft->Contents)) { + if (!It.second.canApplyTo(*Draft->Contents)) { ++InvalidFileCount; LastInvalidFile = It.first(); } @@ -539,7 +539,7 @@ llvm::Optional WithOffsetEncoding; if (Opts.Encoding) WithOffsetEncoding.emplace(kCurrentOffsetEncoding, *Opts.Encoding); - Server.emplace(*CDB, TFS, Opts, + Server.emplace(*CDB, TFS, DraftMgr.getDraftFS(), Opts, static_cast(this)); } applyConfiguration(Params.initializationOptions.ConfigSettings); @@ -711,7 +711,7 @@ return; } - Server->addDocument(File, Draft->Contents, encodeVersion(Draft->Version), + Server->addDocument(File, *Draft->Contents, encodeVersion(Draft->Version), WantDiags, Params.forceRebuild); } @@ -902,7 +902,7 @@ "onDocumentOnTypeFormatting called for non-added file", ErrorCode::InvalidParams)); - Server->formatOnType(File, Code->Contents, Params.position, Params.ch, + Server->formatOnType(File, *Code->Contents, Params.position, Params.ch, std::move(Reply)); } @@ -917,11 +917,11 @@ ErrorCode::InvalidParams)); Server->formatRange( - File, Code->Contents, Params.range, + File, *Code->Contents, Params.range, [Code = Code->Contents, Reply = std::move(Reply)]( llvm::Expected Result) mutable { if (Result) - Reply(replacementsToEdits(Code, Result.get())); + Reply(replacementsToEdits(*Code, Result.get())); else Reply(Result.takeError()); }); @@ -937,11 +937,11 @@ "onDocumentFormatting called for non-added file", ErrorCode::InvalidParams)); - Server->formatFile(File, Code->Contents, + Server->formatFile(File, *Code->Contents, [Code = Code->Contents, Reply = std::move(Reply)]( llvm::Expected Result) mutable { if (Result) - Reply(replacementsToEdits(Code, Result.get())); + Reply(replacementsToEdits(*Code, Result.get())); else Reply(Result.takeError()); }); @@ -1484,7 +1484,8 @@ BackgroundContext(Context::current().clone()), Transp(Transp), MsgHandler(new MessageHandler(*this)), TFS(TFS), SupportedSymbolKinds(defaultSymbolKinds()), - SupportedCompletionItemKinds(defaultCompletionItemKinds()), Opts(Opts) { + SupportedCompletionItemKinds(defaultCompletionItemKinds()), DraftMgr(TFS), + Opts(Opts) { if (Opts.ConfigProvider) { assert(!Opts.ContextProvider && "Only one of ConfigProvider and ContextProvider allowed!"); @@ -1591,14 +1592,14 @@ auto Code = DraftMgr.getDraft(Params.textDocument.uri.file()); if (!Code) return true; // completion code will log the error for untracked doc. - auto Offset = positionToOffset(Code->Contents, Params.position, + auto Offset = positionToOffset(*Code->Contents, Params.position, /*AllowColumnsBeyondLineLength=*/false); if (!Offset) { vlog("could not convert position '{0}' to offset for file '{1}'", Params.position, Params.textDocument.uri.file()); return true; } - return allowImplicitCompletion(Code->Contents, *Offset); + return allowImplicitCompletion(*Code->Contents, *Offset); } void ClangdLSPServer::onDiagnosticsReady(PathRef File, llvm::StringRef Version, @@ -1718,7 +1719,7 @@ for (const Path &FilePath : DraftMgr.getActiveFiles()) if (Filter(FilePath)) if (auto Draft = DraftMgr.getDraft(FilePath)) // else disappeared in race? - Server->addDocument(FilePath, std::move(Draft->Contents), + Server->addDocument(FilePath, *Draft->Contents, encodeVersion(Draft->Version), WantDiagnostics::Auto); } diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -156,7 +156,12 @@ /// those arguments for subsequent reparses. However, ClangdServer will check /// if compilation arguments changed on calls to forceReparse(). ClangdServer(const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS, - const Options &Opts, Callbacks *Callbacks = nullptr); + const ThreadsafeFS &DirtyFS, const Options &Opts, + Callbacks *Callbacks = nullptr); + + ClangdServer(const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS, + const Options &Opts, Callbacks *Callbacks = nullptr) + : ClangdServer(CDB, TFS, TFS, Opts, Callbacks) {} /// Add a \p File to the list of tracked C++ files or update the contents if /// \p File is already tracked. Also schedules parsing of the AST for it on a @@ -336,6 +341,8 @@ const GlobalCompilationDatabase &CDB; const ThreadsafeFS &TFS; + /// A File system that overlays open documents over the underlying filesystem. + const ThreadsafeFS &DirtyFS; Path ResourceDir; // The index used to look up symbols. This could be: diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -125,9 +125,9 @@ } ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, - const ThreadsafeFS &TFS, const Options &Opts, - Callbacks *Callbacks) - : CDB(CDB), TFS(TFS), + const ThreadsafeFS &TFS, const ThreadsafeFS &DirtyFS, + const Options &Opts, Callbacks *Callbacks) + : CDB(CDB), TFS(TFS), DirtyFS(DirtyFS), DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr), ClangTidyProvider(Opts.ClangTidyProvider), WorkspaceRoot(Opts.WorkspaceRoot), diff --git a/clang-tools-extra/clangd/DraftStore.h b/clang-tools-extra/clangd/DraftStore.h --- a/clang-tools-extra/clangd/DraftStore.h +++ b/clang-tools-extra/clangd/DraftStore.h @@ -11,6 +11,7 @@ #include "Protocol.h" #include "support/Path.h" +#include "support/ThreadsafeFS.h" #include "clang/Basic/LLVM.h" #include "llvm/ADT/StringMap.h" #include @@ -20,6 +21,20 @@ namespace clang { namespace clangd { +/// A Reference counted string that is used to hold the contents of open files. +class RefCntString : public llvm::ThreadSafeRefCountedBase { +public: + RefCntString(std::string Str) : Data(std::move(Str)) {} + + StringRef str() const { return Data; } + + operator const std::string &() const { return Data; } + operator StringRef() const { return str(); } + +private: + std::string Data; +}; + /// A thread-safe container for files opened in a workspace, addressed by /// filenames. The contents are owned by the DraftStore. This class supports /// both whole and incremental updates of the documents. @@ -28,10 +43,14 @@ class DraftStore { public: struct Draft { - std::string Contents; + llvm::IntrusiveRefCntPtr Contents; int64_t Version = -1; }; + DraftStore(const ThreadsafeFS &BaseFS); + + DraftStore() = default; + /// \return Contents of the stored document. /// For untracked files, a llvm::None is returned. llvm::Optional getDraft(PathRef File) const; @@ -59,9 +78,20 @@ /// Remove the draft from the store. void removeDraft(PathRef File); + const ThreadsafeFS &getDraftFS() const { + assert(DraftFS && "No draft fs has been set up"); + return *DraftFS; + } + private: + struct DraftAndTime { + Draft Draft; + llvm::sys::TimePoint<> MTime; + }; + class DraftstoreFS; + std::unique_ptr DraftFS; mutable std::mutex Mutex; - llvm::StringMap Drafts; + llvm::StringMap Drafts; }; } // namespace clangd diff --git a/clang-tools-extra/clangd/DraftStore.cpp b/clang-tools-extra/clangd/DraftStore.cpp --- a/clang-tools-extra/clangd/DraftStore.cpp +++ b/clang-tools-extra/clangd/DraftStore.cpp @@ -9,7 +9,9 @@ #include "DraftStore.h" #include "SourceCode.h" #include "support/Logger.h" +#include "llvm/Support/Chrono.h" #include "llvm/Support/Errc.h" +#include namespace clang { namespace clangd { @@ -21,7 +23,7 @@ if (It == Drafts.end()) return None; - return It->second; + return It->second.Draft; } std::vector DraftStore::getActiveFiles() const { @@ -47,14 +49,19 @@ } } +static void updateTime(llvm::sys::TimePoint<> &Time) { + Time = llvm::sys::TimePoint<>::clock::now(); +} + int64_t DraftStore::addDraft(PathRef File, llvm::Optional Version, llvm::StringRef Contents) { std::lock_guard Lock(Mutex); - Draft &D = Drafts[File]; - updateVersion(D, Version); - D.Contents = Contents.str(); - return D.Version; + DraftAndTime &D = Drafts[File]; + updateVersion(D.Draft, Version); + updateTime(D.MTime); + D.Draft.Contents = llvm::makeIntrusiveRefCnt(Contents.str()); + return D.Draft.Version; } llvm::Expected DraftStore::updateDraft( @@ -68,8 +75,8 @@ "Trying to do incremental update on non-added document: {0}", File); } - Draft &D = EntryIt->second; - std::string Contents = EntryIt->second.Contents; + DraftAndTime &D = EntryIt->second; + std::string Contents = *D.Draft.Contents; for (const TextDocumentContentChangeEvent &Change : Changes) { if (!Change.range) { @@ -120,9 +127,11 @@ Contents = std::move(NewContents); } - updateVersion(D, Version); - D.Contents = std::move(Contents); - return D; + updateVersion(D.Draft, Version); + updateTime(D.MTime); + D.Draft.Contents = + llvm::makeIntrusiveRefCnt(std::move(Contents)); + return D.Draft; } void DraftStore::removeDraft(PathRef File) { @@ -131,5 +140,175 @@ Drafts.erase(File); } +namespace { + +class RefCntStringMemBuffer : public llvm::MemoryBuffer { +public: + static std::unique_ptr + create(IntrusiveRefCntPtr Data, StringRef Name) { + // Allocate space for the FileContentMemBuffer and its name with null + // terminator. + static_assert(alignof(RefCntStringMemBuffer) <= sizeof(void *), + "Alignment requirements can't be greater than pointer"); + auto MemSize = sizeof(RefCntStringMemBuffer) + Name.size() + 1; + return std::unique_ptr(new (operator new( + MemSize)) RefCntStringMemBuffer(std::move(Data), Name)); + } + + BufferKind getBufferKind() const override { + return MemoryBuffer::MemoryBuffer_Malloc; + } + + StringRef getBufferIdentifier() const override { + return StringRef(nameBegin(), NameSize); + } + +private: + RefCntStringMemBuffer(IntrusiveRefCntPtr Data, StringRef Name) + : BufferContents(std::move(Data)), NameSize(Name.size()) { + MemoryBuffer::init(BufferContents->str().begin(), + BufferContents->str().end(), true); + memcpy(nameBegin(), Name.begin(), Name.size()); + nameBegin()[Name.size()] = '\0'; + } + + char *nameBegin() { return (char *)&this[1]; } + const char *nameBegin() const { return (const char *)&this[1]; } + + IntrusiveRefCntPtr BufferContents; + size_t NameSize; +}; + +class DraftStoreFile : public llvm::vfs::File { +public: + struct FileData { + IntrusiveRefCntPtr Contents; + llvm::sys::fs::UniqueID UID; + llvm::sys::TimePoint<> MTime; + }; + DraftStoreFile(std::string RequestedName, FileData Data) + : RequestedName(std::move(RequestedName)), Data(std::move(Data)) {} + + llvm::ErrorOr status() override { + return llvm::vfs::Status( + RequestedName, Data.UID, Data.MTime, 0, 0, Data.Contents->str().size(), + llvm::sys::fs::file_type::regular_file, llvm::sys::fs::all_all); + } + + llvm::ErrorOr> + getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, + bool IsVolatile) override { + return RefCntStringMemBuffer::create(Data.Contents, RequestedName); + } + + std::error_code close() override { return {}; } + + ~DraftStoreFile() override { close(); } + +private: + std::string RequestedName; + FileData Data; +}; + +class DraftStoreFileSystem : public llvm::vfs::FileSystem { +public: + DraftStoreFileSystem(llvm::StringMap Contents) + : Contents(std::move(Contents)) {} + + llvm::ErrorOr status(const Twine &Path) override { + SmallString<256> PathStorage; + Path.toVector(PathStorage); + auto EC = makeAbsolute(PathStorage); + assert(!EC); + (void)EC; + auto Iter = Contents.find(PathStorage); + if (Iter == Contents.end()) + return llvm::errc::no_such_file_or_directory; + return DraftStoreFile(std::string(PathStorage), Iter->getValue()).status(); + } + + llvm::ErrorOr> + openFileForRead(const Twine &Path) override { + SmallString<256> PathStorage; + Path.toVector(PathStorage); + auto EC = makeAbsolute(PathStorage); + assert(!EC); + (void)EC; + auto Iter = Contents.find(PathStorage); + if (Iter == Contents.end()) + return llvm::errc::no_such_file_or_directory; + return std::make_unique(std::string(PathStorage), + Iter->getValue()); + } + + llvm::ErrorOr getCurrentWorkingDirectory() const override { + return WorkingDirectory; + } + + std::error_code setCurrentWorkingDirectory(const Twine &P) override { + SmallString<128> Path; + P.toVector(Path); + + // Fix up relative paths. This just prepends the current working + // directory. + std::error_code EC = makeAbsolute(Path); + assert(!EC); + (void)EC; + + llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true); + + if (!Path.empty()) + WorkingDirectory = std::string(Path); + return {}; + } + llvm::vfs::directory_iterator dir_begin(const Twine &Dir, + std::error_code &EC) override { + EC = llvm::errc::no_such_file_or_directory; + return llvm::vfs::directory_iterator(); + } + +private: + std::string WorkingDirectory; + llvm::StringMap Contents; +}; + +} // namespace + +class DraftStore::DraftstoreFS : public ThreadsafeFS { +public: + DraftstoreFS(const ThreadsafeFS &Base, const DraftStore &DS) + : Base(Base), DS(DS) {} + + llvm::IntrusiveRefCntPtr viewImpl() const override { + auto BaseView = Base.view(llvm::None); + llvm::StringMap Contents; + std::lock_guard Guard(DS.Mutex); + for (const auto &KV : DS.Drafts) { + // Query the base filesystem for file uniqueids. + auto BaseStatus = BaseView->status(KV.getKey()); + if (!BaseStatus) { + elog("Couldn't find file {0} when building DirtyFS", KV.getKey()); + continue; + } + // log("Adding dirty file {0} to dirty buffer", KV.getKey()); + Contents.insert(std::make_pair( + KV.getKey(), DraftStoreFile::FileData{KV.getValue().Draft.Contents, + BaseStatus->getUniqueID(), + KV.getValue().MTime})); + } + auto OFS = llvm::makeIntrusiveRefCnt( + std::move(BaseView)); + OFS->pushOverlay( + llvm::makeIntrusiveRefCnt(std::move(Contents))); + return OFS; + } + +private: + const ThreadsafeFS &Base; + const DraftStore &DS; +}; + +DraftStore::DraftStore(const ThreadsafeFS &BaseFS) + : DraftFS(std::make_unique(BaseFS, *this)) {} } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/DraftStoreTests.cpp b/clang-tools-extra/clangd/unittests/DraftStoreTests.cpp --- a/clang-tools-extra/clangd/unittests/DraftStoreTests.cpp +++ b/clang-tools-extra/clangd/unittests/DraftStoreTests.cpp @@ -52,8 +52,8 @@ llvm::Expected Result = DS.updateDraft(Path, llvm::None, {Event}); ASSERT_TRUE(!!Result); - EXPECT_EQ(Result->Contents, SrcAfter.code()); - EXPECT_EQ(DS.getDraft(Path)->Contents, SrcAfter.code()); + EXPECT_EQ(*Result->Contents, SrcAfter.code()); + EXPECT_EQ(*DS.getDraft(Path)->Contents, SrcAfter.code()); EXPECT_EQ(Result->Version, static_cast(i)); } } @@ -84,8 +84,8 @@ DS.updateDraft(Path, llvm::None, Changes); ASSERT_TRUE(!!Result) << llvm::toString(Result.takeError()); - EXPECT_EQ(Result->Contents, FinalSrc.code()); - EXPECT_EQ(DS.getDraft(Path)->Contents, FinalSrc.code()); + EXPECT_EQ(*Result->Contents, FinalSrc.code()); + EXPECT_EQ(*DS.getDraft(Path)->Contents, FinalSrc.code()); EXPECT_EQ(Result->Version, 1); } @@ -345,7 +345,7 @@ Optional Contents = DS.getDraft(File); EXPECT_TRUE(Contents); - EXPECT_EQ(Contents->Contents, OriginalContents); + EXPECT_EQ(*Contents->Contents, OriginalContents); EXPECT_EQ(Contents->Version, 0); }