[clangd] Implement FuzzyFindRequest JSON (de)serialization

[kbobyrev]

Summary

JSON (de)serialization of FuzzyFindRequest might be useful for both D51090 and D51628. Also, this allows precise logging of the fuzzy find requests.

[sammccall]

clang-tools-extra/clangd/CodeComplete.cpp

1384

{0:2} for prettyprinting

clang-tools-extra/clangd/index/Index.cpp

198

Can we explain this in simpler semantic terms?
e.g. // A huge limit means no limit, leave it out.

clang-tools-extra/clangd/index/Index.h

258

please fix your editor to only format changed lines

[kbobyrev]

Revert unrelated formatting changes, simplify the comment.

[yroux]

Hi Kirill,

This commit broke ARMV8 buildbots, logs are available here:
http://lab.llvm.org:8011/builders/clang-cmake-armv8-quick/builds/6493/steps/build%20stage%201/logs/stdio

Thanks
Yvan

[kbobyrev]

In D51852#1228883, @yroux wrote:

Hi Kirill,

This commit broke ARMV8 buildbots, logs are available here:
http://lab.llvm.org:8011/builders/clang-cmake-armv8-quick/builds/6493/steps/build%20stage%201/logs/stdio

Thanks
Yvan

Hi! Apologies for inconvenience, https://reviews.llvm.org/rL341832 should solve the issue.

clang-tools-extra/clangd/CodeComplete.cpp

SymbolSlab queryIndex() {
     FuzzyFindRequest Req;
     if (Opts.Limit)
       Req.MaxCandidateCount = Opts.Limit;
     Req.Query = Filter->pattern();
     Req.RestrictForCodeCompletion = true;
     Req.Scopes = QueryScopes;
     // FIXME: we should send multiple weighted paths here.
     Req.ProximityPaths.push_back(FileName);
-    vlog("Code complete: fuzzyFind(\"{0}\", scopes=[{1}])", Req.Query,
-         llvm::join(Req.Scopes.begin(), Req.Scopes.end(), ","));
+    vlog("Code complete: fuzzyFind({0})", toJSON(Req));
 
     if (SpecFuzzyFind)
       SpecFuzzyFind->NewReq = Req;
     if (SpecFuzzyFind && SpecFuzzyFind->Result.valid() && (*SpecReq == Req)) {
       vlog("Code complete: speculative fuzzy request matches the actual index "
            "request. Waiting for the speculative index results.");
       SPAN_ATTACH(Tracer, "Speculative results", true);
 

clang-tools-extra/clangd/index/Index.h

 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/StringSaver.h"
 #include <array>
+#include <limits>
 #include <mutex>
 #include <string>
 #include <tuple>
 
 namespace clang {
 namespace clangd {
 
 struct SymbolLocation {
enum SymbolFlag : uint8_t {
     IndexedForCodeCompletion = 1 << 0,
     /// Indicates if the symbol is deprecated.
     Deprecated = 1 << 1,
   };
 
   SymbolFlag Flags = SymbolFlag::None;
   /// FIXME: also add deprecation message and fixit?
 };
-inline Symbol::SymbolFlag  operator|(Symbol::SymbolFlag A, Symbol::SymbolFlag  B) {
+inline Symbol::SymbolFlag operator|(Symbol::SymbolFlag A,
+                                    Symbol::SymbolFlag B) {
   return static_cast<Symbol::SymbolFlag>(static_cast<uint8_t>(A) |
                                          static_cast<uint8_t>(B));
 }
-inline Symbol::SymbolFlag &operator|=(Symbol::SymbolFlag &A, Symbol::SymbolFlag B) {
+inline Symbol::SymbolFlag &operator|=(Symbol::SymbolFlag &A,
+                                      Symbol::SymbolFlag B) {
   return A = A | B;
 }
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Symbol &S);
 raw_ostream &operator<<(raw_ostream &, Symbol::SymbolFlag);
 
 // Invokes Callback with each StringRef& contained in the Symbol.
 // Useful for deduplicating backing strings.
 template <typename Callback> void visitStrings(Symbol &S, const Callback &CB) {
struct FuzzyFindRequest {
   /// (e.g. namespaces) excluding nested scopes. For example, if a scope "xyz::"
   /// is provided, the matched symbols must be defined in namespace xyz but not
   /// namespace xyz::abc.
   ///
   /// The global scope is "", a top level scope is "foo::", etc.
   std::vector<std::string> Scopes;
   /// \brief The number of top candidates to return. The index may choose to
   /// return more than this, e.g. if it doesn't know which candidates are best.
-  size_t MaxCandidateCount = UINT_MAX;
+  size_t MaxCandidateCount = std::numeric_limits<size_t>::max();
   /// If set to true, only symbols for completion support will be considered.
   bool RestrictForCodeCompletion = false;
   /// Contextually relevant files (e.g. the file we're code-completing in).
   /// Paths should be absolute.
   std::vector<std::string> ProximityPaths;
 
   bool operator==(const FuzzyFindRequest &Req) const {
     return std::tie(Query, Scopes, MaxCandidateCount, RestrictForCodeCompletion,
                     ProximityPaths) ==
            std::tie(Req.Query, Req.Scopes, Req.MaxCandidateCount,
                     Req.RestrictForCodeCompletion, Req.ProximityPaths);
   }
   bool operator!=(const FuzzyFindRequest &Req) const { return !(*this == Req); }
 };
+bool fromJSON(const llvm::json::Value &Value, FuzzyFindRequest &Request);
+llvm::json::Value toJSON(const FuzzyFindRequest &Request);
 
 struct LookupRequest {
   llvm::DenseSet<SymbolID> IDs;
 };
 
 struct RefsRequest {
   llvm::DenseSet<SymbolID> IDs;
   RefKind Filter = RefKind::All;

clang-tools-extra/clangd/index/Index.cpp

float quality(const Symbol &S) {
   // This avoids a sharp gradient for tail symbols, and also neatly avoids the
   // question of whether 0 references means a bad symbol or missing data.
   if (S.References < 3)
     return 1;
   return std::log(S.References);
 }
 
 SymbolSlab::const_iterator SymbolSlab::find(const SymbolID &ID) const {
-  auto It = std::lower_bound(Symbols.begin(), Symbols.end(), ID,
-                             [](const Symbol &S, const SymbolID &I) {
-                               return S.ID < I;
-                             });
+  auto It = std::lower_bound(
+      Symbols.begin(), Symbols.end(), ID,
+      [](const Symbol &S, const SymbolID &I) { return S.ID < I; });
   if (It != Symbols.end() && It->ID == ID)
     return It;
   return Symbols.end();
 }
 
 // Copy the underlying data of the symbol into the owned arena.
 static void own(Symbol &S, llvm::UniqueStringSaver &Strings) {
   visitStrings(S, [&](StringRef &V) { V = Strings.save(V); });
std::shared_ptr<SymbolIndex> Pin;
     this->Index = std::move(Index);
   }
 }
 std::shared_ptr<SymbolIndex> SwapIndex::snapshot() const {
   std::lock_guard<std::mutex> Lock(Mutex);
   return Index;
 }
 
+bool fromJSON(const llvm::json::Value &Parameters, FuzzyFindRequest &Request) {
+  json::ObjectMapper O(Parameters);
+  llvm::Optional<int64_t> MaxCandidateCount;
+  bool OK =
+      O && O.map("Query", Request.Query) && O.map("Scopes", Request.Scopes) &&
+      O.map("RestrictForCodeCompletion", Request.RestrictForCodeCompletion) &&
+      O.map("ProximityPaths", Request.ProximityPaths) &&
+      O.map("MaxCandidateCount", MaxCandidateCount);
+  if (MaxCandidateCount)
+    Request.MaxCandidateCount = MaxCandidateCount.getValue();
+  return OK;
+}
+
+llvm::json::Value toJSON(const FuzzyFindRequest &Request) {
+  auto Result = json::Object{
+      {"Query", Request.Query},
+      {"Scopes", json::Array{Request.Scopes}},
+      {"RestrictForCodeCompletion", Request.RestrictForCodeCompletion},
+      {"ProximityPaths", json::Array{Request.ProximityPaths}},
+  };
+  // Typically, MaxCandidateCount is either maximum size_t value or something
+  // small. Since LLVM JSON serializer only supports int64_t, try to cast to
+  // int64_t and make this field "optional".
+  if (Request.MaxCandidateCount <= std::numeric_limits<int64_t>::max())
+    Result["MaxCandidateCount"] =
+        static_cast<int64_t>(Request.MaxCandidateCount);
+  return Result;
+}
+
 bool SwapIndex::fuzzyFind(const FuzzyFindRequest &R,
                           llvm::function_ref<void(const Symbol &)> CB) const {
   return snapshot()->fuzzyFind(R, CB);
 }
 void SwapIndex::lookup(const LookupRequest &R,
                        llvm::function_ref<void(const Symbol &)> CB) const {
   return snapshot()->lookup(R, CB);
 }