diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp --- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp +++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp @@ -32,6 +32,9 @@ llvm::cl::opt ExecCommand("c", llvm::cl::desc("Command to execute and then exit")); +llvm::cl::opt ProjectRoot("project-root", + llvm::cl::desc("Path to the project")); + static constexpr char Overview[] = R"( This is an **experimental** interactive tool to process user-provided search queries over given symbol collection obtained via clangd-indexer. The @@ -326,7 +329,8 @@ std::unique_ptr openIndex(llvm::StringRef Index) { return Index.startswith("remote:") - ? remote::getClient(Index.drop_front(strlen("remote:"))) + ? remote::getClient(Index.drop_front(strlen("remote:")), + ProjectRoot) : loadIndex(Index, /*UseDex=*/true); } diff --git a/clang-tools-extra/clangd/index/remote/Client.h b/clang-tools-extra/clangd/index/remote/Client.h --- a/clang-tools-extra/clangd/index/remote/Client.h +++ b/clang-tools-extra/clangd/index/remote/Client.h @@ -10,6 +10,7 @@ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H #include "index/Index.h" +#include "llvm/ADT/StringRef.h" namespace clang { namespace clangd { @@ -22,7 +23,8 @@ /// /// \returns nullptr if the address is not resolved during the function call or /// if the project was compiled without Remote Index support. -std::unique_ptr getClient(llvm::StringRef Address); +std::unique_ptr getClient(llvm::StringRef Address, + llvm::StringRef PathPrefix); } // namespace remote } // namespace clangd diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp --- a/clang-tools-extra/clangd/index/remote/Client.cpp +++ b/clang-tools-extra/clangd/index/remote/Client.cpp @@ -14,6 +14,7 @@ #include "marshalling/Marshalling.h" #include "support/Logger.h" #include "support/Trace.h" +#include "llvm/ADT/StringRef.h" namespace clang { namespace clangd { @@ -44,7 +45,7 @@ FinalResult = Reply.final_result(); continue; } - auto Sym = fromProtobuf(Reply.stream_result(), &Strings); + auto Sym = fromProtobuf(Reply.stream_result(), &Strings, PathPrefix); if (!Sym) elog("Received invalid {0}", ReplyT::descriptor()->name()); Callback(*Sym); @@ -54,8 +55,9 @@ } public: - IndexClient(std::shared_ptr Channel) - : Stub(remote::SymbolIndex::NewStub(Channel)) {} + IndexClient(std::shared_ptr Channel, + llvm::StringRef PathPrefix) + : Stub(remote::SymbolIndex::NewStub(Channel)), PathPrefix(PathPrefix) {} void lookup(const clangd::LookupRequest &Request, llvm::function_ref Callback) const { @@ -84,15 +86,18 @@ private: std::unique_ptr Stub; + llvm::StringRef PathPrefix; }; } // namespace -std::unique_ptr getClient(llvm::StringRef Address) { +std::unique_ptr getClient(llvm::StringRef Address, + llvm::StringRef PathPrefix) { const auto Channel = grpc::CreateChannel(Address.str(), grpc::InsecureChannelCredentials()); Channel->GetState(true); - return std::unique_ptr(new IndexClient(Channel)); + return std::unique_ptr( + new IndexClient(Channel, PathPrefix)); } } // namespace remote diff --git a/clang-tools-extra/clangd/index/remote/Index.proto b/clang-tools-extra/clangd/index/remote/Index.proto --- a/clang-tools-extra/clangd/index/remote/Index.proto +++ b/clang-tools-extra/clangd/index/remote/Index.proto @@ -99,7 +99,10 @@ message SymbolLocation { Position start = 1; Position end = 2; - string file_uri = 3; + // clangd::SymbolLocation storees FileURI, but the protocol transmits only a + // part of its body. Because paths are different on the remote and local + // machines they will be translated in the marshalling layer. + string file_path = 3; } message Position { diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h --- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h +++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h @@ -15,6 +15,7 @@ #include "Index.pb.h" #include "index/Index.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/StringSaver.h" namespace clang { @@ -22,17 +23,29 @@ namespace remote { clangd::FuzzyFindRequest fromProtobuf(const FuzzyFindRequest *Request); +/// Deserializes Clangd symbols and translates relative paths into +/// machine-native URIs. llvm::Optional fromProtobuf(const Symbol &Message, - llvm::UniqueStringSaver *Strings); + llvm::UniqueStringSaver *Strings, + llvm::StringRef ProjectRoot, + llvm::StringRef Scheme = "file"); llvm::Optional fromProtobuf(const Ref &Message, - llvm::UniqueStringSaver *Strings); + llvm::UniqueStringSaver *Strings, + llvm::StringRef ProjectRoot, + llvm::StringRef Scheme = "file"); LookupRequest toProtobuf(const clangd::LookupRequest &From); FuzzyFindRequest toProtobuf(const clangd::FuzzyFindRequest &From); RefsRequest toProtobuf(const clangd::RefsRequest &From); -Ref toProtobuf(const clangd::Ref &From); -Symbol toProtobuf(const clangd::Symbol &From); +/// Serializes Clangd and strips \p IndexedProjectRoot from the file paths +/// because they are specific to the indexing machine. For example, +/// "/remote/machine/project/lib/HelloWorld.cpp" will be translated into +/// "/lib/HelloWorld.cpp" because the absolute path would be different on client +/// machine. +Ref toProtobuf(const clangd::Ref &From, llvm::StringRef IndexedProjectRoot); +Symbol toProtobuf(const clangd::Symbol &From, + llvm::StringRef IndexedProjectRoot); } // namespace remote } // namespace clangd diff --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp --- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp +++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp @@ -16,7 +16,10 @@ #include "index/SymbolOrigin.h" #include "support/Logger.h" #include "clang/Index/IndexSymbol.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" #include "llvm/Support/StringSaver.h" namespace clang { @@ -25,6 +28,41 @@ namespace { +/// Translates \p RelativePath into the absolute path and builds URI for the +/// user machine. +std::string relativePathToURI(llvm::StringRef RelativePath, + llvm::StringRef ProjectRoot, + llvm::StringRef Scheme) { + if (RelativePath.empty()) + return ""; + llvm::SmallString<20> FullPath = ProjectRoot; + llvm::sys::path::append(FullPath, RelativePath); + auto Result = URI::create(FullPath, Scheme); + if (!Result) { + elog("Can not create URI given absolute path {} with scheme {}: {}", + FullPath, Scheme, Result.takeError()); + return ""; + } + return static_cast(Result->toString()); +} + +/// Strips \p IndexedProjectRoot from \p URI body to get file path that is +/// relative to the project root. The stripped path will be transferred to the +/// client. +std::string relativePath(llvm::StringRef URI, + llvm::StringRef IndexedProjectRoot) { + if (URI.empty()) + return URI.str(); + auto ParsedURI = URI::parse(URI); + if (!ParsedURI) + return ""; + llvm::SmallString<20> Result = ParsedURI->body(); + if (IndexedProjectRoot.empty()) + return static_cast(Result); + llvm::sys::path::replace_path_prefix(Result, IndexedProjectRoot, ""); + return static_cast(Result); +} + clangd::SymbolLocation::Position fromProtobuf(const Position &Message) { clangd::SymbolLocation::Position Result; Result.setColumn(static_cast(Message.column())); @@ -59,19 +97,24 @@ } clangd::SymbolLocation fromProtobuf(const SymbolLocation &Message, - llvm::UniqueStringSaver *Strings) { + llvm::UniqueStringSaver *Strings, + llvm::StringRef ProjectRoot, + llvm::StringRef Scheme) { clangd::SymbolLocation Location; Location.Start = fromProtobuf(Message.start()); Location.End = fromProtobuf(Message.end()); - Location.FileURI = Strings->save(Message.file_uri()).begin(); + Location.FileURI = + Strings->save(relativePathToURI(Message.file_path(), ProjectRoot, Scheme)) + .begin(); return Location; } -SymbolLocation toProtobuf(const clangd::SymbolLocation &Location) { +SymbolLocation toProtobuf(const clangd::SymbolLocation &Location, + llvm::StringRef Prefix) { remote::SymbolLocation Result; *Result.mutable_start() = toProtobuf(Location.Start); *Result.mutable_end() = toProtobuf(Location.End); - *Result.mutable_file_uri() = Location.FileURI; + *Result.mutable_file_path() = relativePath(Location.FileURI, Prefix); return Result; } @@ -108,7 +151,9 @@ } llvm::Optional fromProtobuf(const Symbol &Message, - llvm::UniqueStringSaver *Strings) { + llvm::UniqueStringSaver *Strings, + llvm::StringRef ProjectRoot, + llvm::StringRef Scheme) { if (!Message.has_info() || !Message.has_definition() || !Message.has_canonical_declarattion()) { elog("Cannot convert Symbol from Protobuf: {}", Message.ShortDebugString()); @@ -125,9 +170,10 @@ Result.SymInfo = fromProtobuf(Message.info()); Result.Name = Message.name(); Result.Scope = Message.scope(); - Result.Definition = fromProtobuf(Message.definition(), Strings); - Result.CanonicalDeclaration = - fromProtobuf(Message.canonical_declarattion(), Strings); + Result.Definition = + fromProtobuf(Message.definition(), Strings, ProjectRoot, Scheme); + Result.CanonicalDeclaration = fromProtobuf(Message.canonical_declarattion(), + Strings, ProjectRoot, Scheme); Result.References = Message.references(); Result.Origin = static_cast(Message.origin()); Result.Signature = Message.signature(); @@ -144,13 +190,16 @@ } llvm::Optional fromProtobuf(const Ref &Message, - llvm::UniqueStringSaver *Strings) { + llvm::UniqueStringSaver *Strings, + llvm::StringRef ProjectRoot, + llvm::StringRef Scheme) { if (!Message.has_location()) { elog("Cannot convert Ref from Protobuf: {}", Message.ShortDebugString()); return llvm::None; } clangd::Ref Result; - Result.Location = fromProtobuf(Message.location(), Strings); + Result.Location = + fromProtobuf(Message.location(), Strings, ProjectRoot, Scheme); Result.Kind = static_cast(Message.kind()); return Result; } @@ -188,15 +237,15 @@ return RPCRequest; } -Symbol toProtobuf(const clangd::Symbol &From) { +Symbol toProtobuf(const clangd::Symbol &From, llvm::StringRef ProjectRoot) { Symbol Result; Result.set_id(From.ID.str()); *Result.mutable_info() = toProtobuf(From.SymInfo); Result.set_name(From.Name.str()); - *Result.mutable_definition() = toProtobuf(From.Definition); + *Result.mutable_definition() = toProtobuf(From.Definition, ProjectRoot); Result.set_scope(From.Scope.str()); *Result.mutable_canonical_declarattion() = - toProtobuf(From.CanonicalDeclaration); + toProtobuf(From.CanonicalDeclaration, ProjectRoot); Result.set_references(From.References); Result.set_origin(static_cast(From.Origin)); Result.set_signature(From.Signature.str()); @@ -214,10 +263,10 @@ return Result; } -Ref toProtobuf(const clangd::Ref &From) { +Ref toProtobuf(const clangd::Ref &From, llvm::StringRef ProjectRoot) { Ref Result; Result.set_kind(static_cast(From.Kind)); - *Result.mutable_location() = toProtobuf(From.Location); + *Result.mutable_location() = toProtobuf(From.Location, ProjectRoot); return Result; } diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp --- a/clang-tools-extra/clangd/index/remote/server/Server.cpp +++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -31,6 +31,10 @@ llvm::cl::opt IndexPath(llvm::cl::desc(""), llvm::cl::Positional, llvm::cl::Required); +llvm::cl::opt + IndexPrefixPath(llvm::cl::desc(""), + llvm::cl::Positional, llvm::cl::Required); + llvm::cl::opt ServerAddress( "server-address", llvm::cl::init("0.0.0.0:50051"), llvm::cl::desc("Address of the invoked server. Defaults to 0.0.0.0:50051")); @@ -57,7 +61,7 @@ } Index->lookup(Req, [&](const clangd::Symbol &Sym) { LookupReply NextMessage; - *NextMessage.mutable_stream_result() = toProtobuf(Sym); + *NextMessage.mutable_stream_result() = toProtobuf(Sym, IndexPrefixPath); Reply->Write(NextMessage); }); LookupReply LastMessage; @@ -72,7 +76,7 @@ const auto Req = fromProtobuf(Request); bool HasMore = Index->fuzzyFind(Req, [&](const clangd::Symbol &Sym) { FuzzyFindReply NextMessage; - *NextMessage.mutable_stream_result() = toProtobuf(Sym); + *NextMessage.mutable_stream_result() = toProtobuf(Sym, IndexPrefixPath); Reply->Write(NextMessage); }); FuzzyFindReply LastMessage; @@ -92,7 +96,8 @@ } bool HasMore = Index->refs(Req, [&](const clangd::Ref &Reference) { RefsReply NextMessage; - *NextMessage.mutable_stream_result() = toProtobuf(Reference); + *NextMessage.mutable_stream_result() = + toProtobuf(Reference, IndexPrefixPath); Reply->Write(NextMessage); }); RefsReply LastMessage; diff --git a/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp b/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp --- a/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp +++ b/clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp @@ -8,12 +8,14 @@ #include "index/remote/Client.h" #include "support/Logger.h" +#include "llvm/ADT/StringRef.h" namespace clang { namespace clangd { namespace remote { -std::unique_ptr getClient(llvm::StringRef Address) { +std::unique_ptr getClient(llvm::StringRef Address, + llvm::StringRef PathPrefix) { elog("Can't create SymbolIndex client without Remote Index support."); return nullptr; } diff --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp --- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp +++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp @@ -40,8 +40,9 @@ llvm::BumpPtrAllocator Arena; llvm::UniqueStringSaver Strings(Arena); for (auto &Sym : Symbols) { - const auto ProtobufMeessage = toProtobuf(Sym); - const auto SymToProtobufAndBack = fromProtobuf(ProtobufMeessage, &Strings); + const auto ProtobufMessage = toProtobuf(Sym, ""); + const auto SymToProtobufAndBack = fromProtobuf( + ProtobufMessage, &Strings, std::string(testRoot()) + '/', "unittest"); EXPECT_TRUE(SymToProtobufAndBack.hasValue()); EXPECT_EQ(toYAML(Sym), toYAML(*SymToProtobufAndBack)); } @@ -80,12 +81,32 @@ EXPECT_GE(References.numRefs(), 5UL); for (const auto &SymbolWithRefs : References) { for (const auto &Ref : SymbolWithRefs.second) { - const auto RefToProtobufAndBack = fromProtobuf(toProtobuf(Ref), &Strings); + const auto RefToProtobufAndBack = + fromProtobuf(toProtobuf(Ref, ""), &Strings, + std::string(testRoot()) + '/', "unittest"); EXPECT_TRUE(RefToProtobufAndBack.hasValue()); EXPECT_EQ(toYAML(Ref), toYAML(*RefToProtobufAndBack)); } } -} // namespace +} + +TEST(URITranslation, Marshalling) { + llvm::BumpPtrAllocator Arena; + llvm::UniqueStringSaver Strings(Arena); + clangd::Ref Original; + const std::string RemoteIndexPrefix = "/remote/machine/project"; + const std::string RelativePath = "/llvm-project/clang-tools-extra/clangd/" + "unittests/remote/MarshallingTests.cpp"; + const auto URI = URI::createFile(RemoteIndexPrefix + RelativePath); + Original.Location.FileURI = Strings.save(URI.toString()).begin(); + Ref Serialized = toProtobuf(Original, RemoteIndexPrefix); + EXPECT_EQ(Serialized.location().file_path(), RelativePath); + const std::string LocalIndexPrefix = "/local/machine/myproject"; + auto Deserialized = fromProtobuf(Serialized, &Strings, LocalIndexPrefix); + EXPECT_TRUE(Deserialized); + EXPECT_EQ(Deserialized->Location.FileURI, + URI::createFile(LocalIndexPrefix + RelativePath).toString()); +} } // namespace } // namespace remote