[clangd] dexp tool uses llvm::cl to parse its flags.

[sammccall]

Summary

We can use cl::ResetCommandLineParser() to support different types of
command-lines, as long as we're careful about option lifetimes.
(I tried using subcommands, but the error messages were bad)
I found a mostly-reasonable pattern to isolate the fiddly parts.

Added -scope and -limit flags to the find command to demonstrate.
(Note that scope support seems to be broken in dex?)

Fixed symbol lookup to parse symbol IDs.

Caveats:

• with command help (e.g. find -help), you also get some spam about required arguments. This is a bug in llvm::cl, which prints these to errs() rather than the designated stream.

[ilya-biryukov]

A small drive-by comment.

PS it'd be cool to have an interface to cl that does not rely on global state...

clangd/index/dex/dexp/Dexp.cpp

165

Maybe use a named struct?
C++ is powerful, but looks obscure at times...

[kbobyrev]

In D51989#1232099, @ilya-biryukov wrote:

PS it'd be cool to have an interface to cl that does not rely on global state...

Yeah, I'm not sure if that's trivial to do, though.

Thanks for the patch, it looks really good, I don't have any concerns.

clangd/index/dex/dexp/Dexp.cpp

 #include "llvm/LineEditor/LineEditor.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Signals.h"
 
 using clang::clangd::FuzzyFindRequest;
 using clang::clangd::loadIndex;
 using clang::clangd::Symbol;
 using clang::clangd::SymbolIndex;
-using llvm::StringRef;
+using namespace llvm;
 
 namespace {
 
 llvm::cl::opt<std::string>
     SymbolCollection("symbol-collection-file",
                      llvm::cl::desc("Path to the file with symbol collection"),
                      llvm::cl::Positional, llvm::cl::Required);
 
void reportTime(StringRef Name, llvm::function_ref<void()> F) {
   const auto TimerStart = std::chrono::high_resolution_clock::now();
   F();
   const auto TimerStop = std::chrono::high_resolution_clock::now();
   const auto Duration = std::chrono::duration_cast<std::chrono::milliseconds>(
       TimerStop - TimerStart);
   llvm::outs() << llvm::formatv("{0} took {1:ms+n}.\n", Name, Duration);
 }
 
-void fuzzyFind(llvm::StringRef UnqualifiedName, const SymbolIndex &Index) {
+// REPL commands inherit from Command and contain their options as members.
+// Creating a Command populates parser options, parseAndRun() resets them.
+class Command {
+  // By resetting the parser options, we lost the standard -help flag.
+  cl::opt<bool, false, cl::parser<bool>> Help{
+      "help", cl::desc("Display available options"), cl::ValueDisallowed,
+      cl::cat(cl::GeneralCategory)};
+  virtual void run() = 0;
+
+protected:
+  const SymbolIndex *Index;
+
+public:
+  virtual ~Command() = default;
+  virtual void parseAndRun(ArrayRef<const char *> Argv, const char *Overview,
+                           const SymbolIndex &Index) {
+    std::string ParseErrs;
+    llvm::raw_string_ostream OS(ParseErrs);
+    bool Ok =
+        cl::ParseCommandLineOptions(Argv.size(), Argv.data(), Overview, &OS);
+    if (Help.getNumOccurrences() > 0) {
+      // Avoid printing parse errors in this case.
+      // (Well, in theory. A bunch get printed to llvm::errs() regardless!)
+      cl::PrintHelpMessage();
+    } else {
+      outs() << OS.str();
+      if (Ok) {
+        this->Index = &Index;
+        reportTime(Argv[0], [&] { run(); });
+      }
+    }
+    cl::ResetCommandLineParser(); // must do this before opts are destroyed.
+  }
+};
+
+// FIXME(kbobyrev): Ideas for more commands:
+// * find symbol references: print set of reference locations
+// * load/swap/reload index: this would make it possible to get rid of llvm::cl
+//   usages in the tool driver and actually use llvm::cl library in the REPL.
+// * show posting list density histogram (our dump data somewhere so that user
+//   could build one)
+// * show number of tokens of each kind
+// * print out tokens with the most dense posting lists
+// * print out tokens with least dense posting lists
+
+class FuzzyFind : public Command {
+  cl::opt<std::string> Query{
+      "query",
+      cl::Positional,
+      cl::Required,
+      cl::desc("Query string to be fuzzy-matched"),
+  };
+  cl::opt<std::string> Scopes{
+      "scopes",
+      cl::desc("Allowed symbol scopes (comma-separated list)"),
+  };
+  cl::opt<unsigned> Limit{
+      "limit",
+      cl::init(10),
+      cl::desc("Max results to display"),
+  };
+
+  void run() override {
     FuzzyFindRequest Request;
-  Request.Limit = 10;
-  Request.Query = UnqualifiedName;
+    Request.Limit = Limit;
+    Request.Query = Query;
+    if (Scopes.getNumOccurrences() > 0) {
+      llvm::SmallVector<StringRef, 8> Scopes;
+      StringRef(this->Scopes).split(Scopes, ',');
+      Request.Scopes = {Scopes.begin(), Scopes.end()};
+    }
     // FIXME(kbobyrev): Print symbol final scores to see the distribution.
     static const auto OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n";
     llvm::outs() << llvm::formatv(OutputFormat, "Rank", "Symbol ID",
                                   "Symbol Name");
     size_t Rank = 0;
-  Index.fuzzyFind(Request, [&](const Symbol &Sym) {
-    llvm::outs() << llvm::formatv(OutputFormat, Rank++, Sym.ID.str(), Sym.Name);
+    Index->fuzzyFind(Request, [&](const Symbol &Sym) {
+      llvm::outs() << llvm::formatv(OutputFormat, Rank++, Sym.ID.str(),
+                                    Sym.Name);
     });
   }
+};
 
-static const std::string HelpMessage = R"(dexp commands:
-
-> find Name
-
-Constructs fuzzy find request given unqualified symbol name and returns top 10
-symbols retrieved from index.
-
-> lookup SymbolID
-
-Retrieves symbol names given USR.
-)";
-
-void help() { llvm::outs() << HelpMessage; }
+class Lookup : public Command {
+  cl::opt<std::string> ID{
+      "id",
+      cl::Positional,
+      cl::Required,
+      cl::desc("Symbol ID to look up (hex)"),
+  };
+
+  void run() override {
+    auto Raw = fromHex(ID);
+    if (Raw.size() != clang::clangd::SymbolID::RawSize) {
+      llvm::outs() << "invalid SymbolID\n";
+      return;
+    }
 
-void lookup(StringRef USR, const SymbolIndex &Index) {
-  llvm::DenseSet<clang::clangd::SymbolID> IDs{clang::clangd::SymbolID{USR}};
-  clang::clangd::LookupRequest Request{IDs};
+    clang::clangd::LookupRequest Request;
+    Request.IDs = {clang::clangd::SymbolID::fromRaw(Raw)};
     bool FoundSymbol = false;
-  Index.lookup(Request, [&](const Symbol &Sym) {
-    if (!FoundSymbol)
+    Index->lookup(Request, [&](const Symbol &Sym) {
       FoundSymbol = true;
       llvm::outs() << SymbolToYAML(Sym);
     });
     if (!FoundSymbol)
       llvm::outs() << "not found\n";
   }
+};
 
-// FIXME(kbobyrev): Make this an actual REPL: probably use LLVM Command Line
-// library for parsing flags and arguments.
-// FIXME(kbobyrev): Ideas for commands:
-// * symbol lookup: print out symbol in YAML format given SymbolID
-// * find symbol references: print set of reference locations
-// * load/swap/reload index: this would make it possible to get rid of llvm::cl
-//   usages in the tool driver and actually use llvm::cl library in the REPL.
-// * show posting list density histogram (our dump data somewhere so that user
-//   could build one)
-// * show number of tokens of each kind
-// * print out tokens with the most dense posting lists
-// * print out tokens with least dense posting lists
-void dispatch(StringRef Request, const SymbolIndex &Index) {
-  llvm::SmallVector<StringRef, 2> Arguments;
-  Request.split(Arguments, ' ');
-  if (Arguments.empty()) {
-    llvm::outs() << "Request can not be empty.\n";
-    help();
-    return;
-  }
-
-  if (Arguments.front() == "find") {
-    if (Arguments.size() != 2) {
-      llvm::outs() << "find request must specify unqualified symbol name.\n";
-      return;
-    }
-    reportTime("fuzzy find request",
-               [&]() { fuzzyFind(Arguments.back(), Index); });
-  } else if (Arguments.front() == "lookup") {
-    if (Arguments.size() != 2) {
-      llvm::outs() << "lookup request must specify symbol ID .\n";
-      return;
-    }
-    reportTime("lookup request", [&]() { lookup(Arguments.back(), Index); });
-  } else if (Arguments.front() == "help") {
-    help();
-  } else {
-    llvm::outs() << "Unknown command. Try 'help'\n";
-  }
-}
+struct {
+  const char *Name;
+  const char *Description;
+  std::function<std::unique_ptr<Command>()> Implementation;
+} CommandInfo[] = {
+    {"find", "Search for symbols with fuzzyFind", llvm::make_unique<FuzzyFind>},
+    {"lookup", "Dump symbol details by ID", llvm::make_unique<Lookup>},
+};
 
 } // namespace
 
 int main(int argc, const char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+  llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   std::unique_ptr<SymbolIndex> Index;
   reportTime("Dex build", [&]() {
     Index = loadIndex(SymbolCollection, /*URISchemes=*/{},
                       /*UseDex=*/true);
   });
 
   if (!Index) {
     llvm::outs()
         << "ERROR: Please provide a valid path to symbol collection file.\n";
     return -1;
   }
 
   llvm::LineEditor LE("dexp");
 
-  while (llvm::Optional<std::string> Request = LE.readLine())
-    dispatch(Request.getValue(), *Index);
+  while (llvm::Optional<std::string> Request = LE.readLine()) {
+    // Split on spaces and add required null-termination.
+    std::replace(Request->begin(), Request->end(), ' ', '\0');
+    SmallVector<StringRef, 8> Args;
+    StringRef(*Request).split(Args, '\0', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
+    if (Args.empty())
+      continue;
+    if (Args.front() == "help") {
+      outs() << "dexp - Index explorer\nCommands:\n";
+      for (const auto &C : CommandInfo)
+        outs() << llvm::formatv("{0,16} - {1}\n", C.Name, C.Description);
+      outs() << "Get detailed command help with e.g. `find -help`.\n";
+      continue;
+    }
+    SmallVector<const char *, 8> FakeArgv;
+    for (StringRef S : Args)
+      FakeArgv.push_back(S.data()); // Terminated by separator or end of string.
+
+    bool Recognized = false;
+    for (const auto &Cmd : CommandInfo) {
+      if (Cmd.Name == Args.front()) {
+        Recognized = true;
+        Cmd.Implementation()->parseAndRun(FakeArgv, Cmd.Description, *Index);
+        break;
+      }
+    }
+    if (!Recognized)
+      outs() << "Unknown command. Try 'help'.\n";
+  }
 
   return 0;
 }

test/CMakeLists.txt

set(CLANG_TOOLS_TEST_DEPS
   clang-reorder-fields
   find-all-symbols
   modularize
   pp-trace
 
   # These individual tools have no tests, add them here to make them compile
   # together with check-clang-tools, so that we won't break them in the future.
   clangd-indexer
+  dexp
 
   # Unit tests
   ExtraToolsUnitTests
   )
 
 if(CLANG_ENABLE_STATIC_ANALYZER)
   list(APPEND CLANG_TOOLS_TEST_DEPS
     # For the clang-tidy libclang integration test.