[include-fixer] collect the number of times a symbols is found in an indexing run and use it for symbols popularity ranking.

[ioeric]

Summary

[include-fixer] collect the number of times a symbols is found in an
indexing run and use it for symbols popularity ranking.

[bkramer]

include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp

91

Can we remove the set now? 2 copies of all SymbolInfos seems unnecessary.

[ioeric]

• Removed a redundant set of symbols during merging.

[bkramer]

lg

include-fixer/find-all-symbols/SymbolInfo.h

public:
   /// \brief A pair of <ContextType, ContextName>.
   typedef std::pair<ContextType, std::string> Context;
 
   // The default constructor is required by YAML traits in
   // LLVM_YAML_IS_DOCUMENT_LIST_VECTOR.
   SymbolInfo() : Type(SymbolKind::Unknown), LineNumber(-1) {}
 
   SymbolInfo(llvm::StringRef Name, SymbolKind Type, llvm::StringRef FilePath,
-             int LineNumber, const std::vector<Context> &Contexts);
+             int LineNumber, const std::vector<Context> &Contexts,
+             unsigned NumOccurrences = 0);
 
   /// \brief Get symbol name.
   llvm::StringRef getName() const;
 
   /// \brief Get symbol type.
   SymbolKind getSymbolKind() const;
 
   /// \brief Get a relative file path where symbol comes from.
   llvm::StringRef getFilePath() const;
 
   /// \brief Get symbol contexts.
   const std::vector<SymbolInfo::Context> &getContexts() const;
 
   /// \brief Get a 1-based line number of the symbol's declaration.
   int getLineNumber() const;
 
+  /// \brief The number of times this symbol was found during an indexing run.
+  unsigned getNumOccurrences() const;
+
   bool operator<(const SymbolInfo &Symbol) const;
 
   bool operator==(const SymbolInfo &Symbol) const;
 
 private:
   friend struct llvm::yaml::MappingTraits<SymbolInfo>;
 
   /// \brief Identifier name.
private:
   /// "na"} }.
   /// The name of an anonymous namespace is "".
   ///
   /// If the symbol is declared in `TranslationUnitDecl`, it has no context.
   std::vector<Context> Contexts;
 
   /// \brief The 1-based line number of of the symbol's declaration.
   int LineNumber;
+
+  /// \brief The number of times this symbol was found during an indexing
+  /// run. Populated by the reducer and used to rank results.
+  unsigned NumOccurrences;
 };
 
 /// \brief Write SymbolInfos to a stream (YAML format).
 bool WriteSymbolInfosToStream(llvm::raw_ostream &OS,
                               const std::set<SymbolInfo> &Symbols);
 
 /// \brief Read SymbolInfos from a YAML document.
 std::vector<SymbolInfo> ReadSymbolInfosFromYAML(llvm::StringRef Yaml);
 
 } // namespace find_all_symbols
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_INCLUDE_FIXER_FIND_ALL_SYMBOLS_SYMBOLINFO_H

include-fixer/find-all-symbols/SymbolInfo.cpp

 namespace yaml {
 template <> struct MappingTraits<SymbolInfo> {
   static void mapping(IO &io, SymbolInfo &Symbol) {
     io.mapRequired("Name", Symbol.Name);
     io.mapRequired("Contexts", Symbol.Contexts);
     io.mapRequired("FilePath", Symbol.FilePath);
     io.mapRequired("LineNumber", Symbol.LineNumber);
     io.mapRequired("Type", Symbol.Type);
+    io.mapRequired("NumOccurrences", Symbol.NumOccurrences);
   }
 };
 
 template <> struct ScalarEnumerationTraits<ContextType> {
   static void enumeration(IO &io, ContextType &value) {
     io.enumCase(value, "Record", ContextType::Record);
     io.enumCase(value, "Namespace", ContextType::Namespace);
     io.enumCase(value, "EnumDecl", ContextType::EnumDecl);

 } // namespace yaml
 } // namespace llvm
 
 namespace clang {
 namespace find_all_symbols {
 
 SymbolInfo::SymbolInfo(llvm::StringRef Name, SymbolKind Type,
                        llvm::StringRef FilePath, int LineNumber,
-                       const std::vector<Context> &Contexts)
+                       const std::vector<Context> &Contexts,
+                       unsigned NumOccurrences)
     : Name(Name), Type(Type), FilePath(FilePath), Contexts(Contexts),
-      LineNumber(LineNumber) {}
+      LineNumber(LineNumber), NumOccurrences(NumOccurrences) {}
 
 llvm::StringRef SymbolInfo::getName() const { return Name; }
 
 SymbolKind SymbolInfo::getSymbolKind() const { return Type; }
 
 llvm::StringRef SymbolInfo::getFilePath() const { return FilePath; }
 
 const std::vector<SymbolInfo::Context> &SymbolInfo::getContexts() const {
   return Contexts;
 }
 
 int SymbolInfo::getLineNumber() const { return LineNumber; }
 
+unsigned SymbolInfo::getNumOccurrences() const { return NumOccurrences; }
+
 bool SymbolInfo::operator==(const SymbolInfo &Symbol) const {
   return std::tie(Name, Type, FilePath, LineNumber, Contexts) ==
          std::tie(Symbol.Name, Symbol.Type, Symbol.FilePath, Symbol.LineNumber,
                   Symbol.Contexts);
 }
 
 bool SymbolInfo::operator<(const SymbolInfo &Symbol) const {
   return std::tie(Name, Type, FilePath, LineNumber, Contexts) <

include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp

public:
   }
 
 private:
   std::map<std::string, std::set<SymbolInfo>> Symbols;
 };
 
 bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) {
   std::error_code EC;
-  std::set<SymbolInfo> UniqueSymbols;
+  std::map<SymbolInfo, int> SymbolToNumOccurrences;
   std::mutex SymbolMutex;
   auto AddSymbols = [&](ArrayRef<SymbolInfo> Symbols) {
     // Synchronize set accesses.
     std::unique_lock<std::mutex> LockGuard(SymbolMutex);
-    UniqueSymbols.insert(Symbols.begin(), Symbols.end());
+    for (const auto &Symbol : Symbols)
+      ++SymbolToNumOccurrences[Symbol];
   };
 
   // Load all symbol files in MergeDir.
   {
     llvm::ThreadPool Pool;
     for (llvm::sys::fs::directory_iterator Dir(MergeDir, EC), DirEnd;
          Dir != DirEnd && !EC; Dir.increment(EC)) {
       // Parse YAML files in parallel.
bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) {
   }
 
   llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::F_None);
   if (EC) {
     llvm::errs() << "Can't open '" << OutputFile << "': " << EC.message()
                  << '\n';
     return false;
   }
-  WriteSymbolInfosToStream(OS, UniqueSymbols);
+  std::set<SymbolInfo> Result;
+  for (const auto &Entry : SymbolToNumOccurrences) {
+    const auto &Symbol = Entry.first;
+    Result.insert(SymbolInfo(Symbol.getName(), Symbol.getSymbolKind(),
+                             Symbol.getFilePath(), Symbol.getLineNumber(),
+                             Symbol.getContexts(), Entry.second));
+  }
+  WriteSymbolInfosToStream(OS, Result);
   return true;
 }
 
 } // namespace clang
 } // namespace find_all_symbols
 
 int main(int argc, const char **argv) {
   CommonOptionsParser OptionsParser(argc, argv, FindAllSymbolsCategory);

test/include-fixer/Inputs/fake_yaml_db.yaml

 ---
 Name:           foo
 Contexts:
   - ContextType:     Namespace
     ContextName:     a
   - ContextType:     Namespace
     ContextName:     b
 FilePath:        foo.h
 LineNumber:      1
 Type:            Class
+NumOccurrences:  1
 ...
 ---
 Name:           bar
 Contexts:
   - ContextType:     Namespace
     ContextName:     a
   - ContextType:     Namespace
     ContextName:     b
 FilePath:        ../include/bar.h
 LineNumber:      1
 Type:            Class
+NumOccurrences:  1
 ...