[clangd] Improvements to header mapping: more precise parsing of cppreference symbol pages.

[sammccall]

Summary

Previously we were just jumping from the symbol index to the symbol page, and
grabbing all the headers mentioned there. But the page often lists multiple
symbols, and so we got false positives and thus ambiguities (which were dropped).

Now we look at which declarations are for the symbol we want, and prefer headers
listed above that symbol. If there are none, we fall back to the old behavior.

[sammccall]

fix std::chrono regression

[kadircet]

clangd/include-mapping/gen_std.py

90

does this also work with functions? For example floor will be written as:

constexpr ToDuration floor(const duration<Rep, Period>& d);

which does not contain floor as a word. https://en.cppreference.com/w/cpp/chrono/duration/floor

95

why not perform clean-up unconditionally?

[kadircet]

clangd/include-mapping/gen_std.py

60

I think this requires some corresponding changes in clangd/include-mapping/test.py

[kadircet]

clangd/include-mapping/gen_std.py

95

ok I see the reason for that one, https://en.cppreference.com/w/cpp/numeric/math/abs, NVM

[sammccall]

Update tests.

[sammccall]

clangd/include-mapping/gen_std.py

90

which does not contain floor as a word

\b matches between a word character and non-word character.
Both and ( are non-word characters, so we should be fine?

95

This isn't cleanup.

header1
header2
declX

declX is associated with header1 and header2

header1
declY
header2
declX

declX is associated with header2 only. We reset current_headers when we see header2 because it was preceded by a decl. I'll add a comment.

[sammccall]

Add explanatory comment about headers reset.

[sammccall]

rebase

[kadircet]

Thanks!

clangd/CodeComplete.cpp

private:
   mergeResults(const std::vector<CodeCompletionResult> &SemaResults,
                const SymbolSlab &IndexResults,
                const std::vector<RawIdentifier> &IdentifierResults) {
     trace::Span Tracer("Merge and score results");
     std::vector<CompletionCandidate::Bundle> Bundles;
     llvm::DenseMap<size_t, size_t> BundleLookup;
     auto AddToBundles = [&](const CodeCompletionResult *SemaResult,
                             const Symbol *IndexResult,
-                            const RawIdentifier *IdentifierResult = nullptr) {
+                            const RawIdentifier *IdentifierResult) {
       CompletionCandidate C;
       C.SemaResult = SemaResult;
       C.IndexResult = IndexResult;
       C.IdentifierResult = IdentifierResult;
       if (C.IndexResult) {
         C.Name = IndexResult->Name;
         C.RankedIncludeHeaders = getRankedIncludes(*C.IndexResult);
       } else if (C.SemaResult) {
auto CorrespondingIndexResult =
           UsedIndexResults.insert(&*I);
           return &*I;
         }
       }
       return nullptr;
     };
     // Emit all Sema results, merging them with Index results if possible.
     for (auto &SemaResult : SemaResults)
-      AddToBundles(&SemaResult, CorrespondingIndexResult(SemaResult));
+      AddToBundles(&SemaResult, CorrespondingIndexResult(SemaResult), nullptr);
     // Now emit any Index-only results.
     for (const auto &IndexResult : IndexResults) {
       if (UsedIndexResults.count(&IndexResult))
         continue;
-      AddToBundles(/*SemaResult=*/nullptr, &IndexResult);
+      AddToBundles(/*SemaResult=*/nullptr, &IndexResult, nullptr);
     }
     // Emit identifier results.
     for (const auto &Ident : IdentifierResults)
       AddToBundles(/*SemaResult=*/nullptr, /*IndexResult=*/nullptr, &Ident);
     // We only keep the best N results at any time, in "native" format.
     TopN<ScoredBundle, ScoredBundleGreater> Top(
         Opts.Limit == 0 ? std::numeric_limits<size_t>::max() : Opts.Limit);
     for (auto &Bundle : Bundles)

clangd/StdSymbolMap.inc

 SYMBOL(bad_cast, std::, <typeinfo>)
 SYMBOL(bad_exception, std::, <exception>)
 SYMBOL(bad_function_call, std::, <functional>)
 SYMBOL(bad_optional_access, std::, <optional>)
 SYMBOL(bad_typeid, std::, <typeinfo>)
 SYMBOL(bad_variant_access, std::, <variant>)
 SYMBOL(bad_weak_ptr, std::, <memory>)
 SYMBOL(basic_common_reference, std::, <type_traits>)
+SYMBOL(basic_filebuf, std::, <fstream>)
 SYMBOL(basic_fstream, std::, <fstream>)
 SYMBOL(basic_ifstream, std::, <fstream>)
 SYMBOL(basic_ios, std::, <ios>)
 SYMBOL(basic_iostream, std::, <istream>)
+SYMBOL(basic_istream, std::, <istream>)
 SYMBOL(basic_istringstream, std::, <sstream>)
 SYMBOL(basic_ofstream, std::, <fstream>)
+SYMBOL(basic_ostream, std::, <ostream>)
 SYMBOL(basic_ostringstream, std::, <sstream>)
 SYMBOL(basic_osyncstream, std::, <syncstream>)
 SYMBOL(basic_regex, std::, <regex>)
 SYMBOL(basic_streambuf, std::, <streambuf>)
 SYMBOL(basic_string, std::, <string>)
 SYMBOL(basic_string_view, std::, <string_view>)
 SYMBOL(basic_stringbuf, std::, <sstream>)
 SYMBOL(basic_stringstream, std::, <sstream>)

 SYMBOL(clearerr, std::, <cstdio>)
 SYMBOL(clock, std::, <ctime>)
 SYMBOL(clock_t, std::, <ctime>)
 SYMBOL(clog, std::, <iostream>)
 SYMBOL(cmatch, std::, <regex>)
 SYMBOL(codecvt, std::, <locale>)
 SYMBOL(codecvt_base, std::, <locale>)
 SYMBOL(codecvt_byname, std::, <locale>)
+SYMBOL(codecvt_mode, std::, <codecvt>)
 SYMBOL(codecvt_utf16, std::, <codecvt>)
 SYMBOL(codecvt_utf8, std::, <codecvt>)
 SYMBOL(codecvt_utf8_utf16, std::, <codecvt>)
 SYMBOL(collate, std::, <locale>)
 SYMBOL(collate_byname, std::, <locale>)
 SYMBOL(common_comparison_category, std::, <compare>)
 SYMBOL(common_comparison_category_t, std::, <compare>)
 SYMBOL(common_reference, std::, <type_traits>)

 SYMBOL(deca, std::, <ratio>)
 SYMBOL(decay, std::, <type_traits>)
 SYMBOL(decay_t, std::, <type_traits>)
 SYMBOL(deci, std::, <ratio>)
 SYMBOL(declare_no_pointers, std::, <memory>)
 SYMBOL(declare_reachable, std::, <memory>)
 SYMBOL(declval, std::, <utility>)
 SYMBOL(default_delete, std::, <memory>)
+SYMBOL(default_random_engine, std::, <random>)
 SYMBOL(default_searcher, std::, <functional>)
 SYMBOL(defaultfloat, std::, <ios>)
 SYMBOL(defer_lock, std::, <mutex>)
 SYMBOL(defer_lock_t, std::, <mutex>)
 SYMBOL(denorm_absent, std::, <limits>)
 SYMBOL(denorm_indeterminate, std::, <limits>)
 SYMBOL(denorm_present, std::, <limits>)
 SYMBOL(deque, std::, <deque>)
 SYMBOL(destroy, std::, <memory>)
 SYMBOL(destroy_at, std::, <memory>)
 SYMBOL(destroy_n, std::, <memory>)
 SYMBOL(destroying_delete, std::, <new>)
 SYMBOL(destroying_delete_t, std::, <new>)
 SYMBOL(difftime, std::, <ctime>)
 SYMBOL(discard_block_engine, std::, <random>)
 SYMBOL(discrete_distribution, std::, <random>)
 SYMBOL(disjunction, std::, <type_traits>)
 SYMBOL(disjunction_v, std::, <type_traits>)
 SYMBOL(distance, std::, <iterator>)
+SYMBOL(div_t, std::, <cstdlib>)
 SYMBOL(divides, std::, <functional>)
 SYMBOL(domain_error, std::, <stdexcept>)
+SYMBOL(double_t, std::, <cmath>)
 SYMBOL(dynamic_extent, std::, <span>)
 SYMBOL(dynamic_pointer_cast, std::, <memory>)
 SYMBOL(emit_on_flush, std::, <ostream>)
 SYMBOL(empty, std::, <iterator>)
 SYMBOL(enable_if, std::, <type_traits>)
 SYMBOL(enable_if_t, std::, <type_traits>)
 SYMBOL(enable_shared_from_this, std::, <memory>)
 SYMBOL(end, std::, <iterator>)

 SYMBOL(exclusive_scan, std::, <numeric>)
 SYMBOL(exit, std::, <cstdlib>)
 SYMBOL(exp2, std::, <cmath>)
 SYMBOL(expm1, std::, <cmath>)
 SYMBOL(exponential_distribution, std::, <random>)
 SYMBOL(extent, std::, <type_traits>)
 SYMBOL(extent_v, std::, <type_traits>)
 SYMBOL(extreme_value_distribution, std::, <random>)
+SYMBOL(fabs, std::, <cmath>)
 SYMBOL(false_type, std::, <type_traits>)
 SYMBOL(fclose, std::, <cstdio>)
 SYMBOL(fdim, std::, <cmath>)
 SYMBOL(feclearexcept, std::, <cfenv>)
 SYMBOL(fegetenv, std::, <cfenv>)
 SYMBOL(fegetexceptflag, std::, <cfenv>)
 SYMBOL(fegetround, std::, <cfenv>)
 SYMBOL(feholdexcept, std::, <cfenv>)

 SYMBOL(feupdateenv, std::, <cfenv>)
 SYMBOL(fexcept_t, std::, <cfenv>)
 SYMBOL(fflush, std::, <cstdio>)
 SYMBOL(fgetc, std::, <cstdio>)
 SYMBOL(fgetpos, std::, <cstdio>)
 SYMBOL(fgets, std::, <cstdio>)
 SYMBOL(fgetwc, std::, <cwchar>)
 SYMBOL(fgetws, std::, <cwchar>)
+SYMBOL(filebuf, std::, <streambuf>)
 SYMBOL(fill, std::, <algorithm>)
 SYMBOL(fill_n, std::, <algorithm>)
 SYMBOL(find, std::, <algorithm>)
 SYMBOL(find_end, std::, <algorithm>)
 SYMBOL(find_first_of, std::, <algorithm>)
 SYMBOL(find_if, std::, <algorithm>)
 SYMBOL(find_if_not, std::, <algorithm>)
 SYMBOL(fisher_f_distribution, std::, <random>)
 SYMBOL(fixed, std::, <ios>)
 SYMBOL(float_denorm_style, std::, <limits>)
 SYMBOL(float_round_style, std::, <limits>)
+SYMBOL(float_t, std::, <cmath>)
 SYMBOL(floor, std::, <cmath>)
 SYMBOL(floor2, std::, <bit>)
 SYMBOL(flush, std::, <ostream>)
 SYMBOL(flush_emit, std::, <ostream>)
 SYMBOL(fma, std::, <cmath>)
 SYMBOL(fmax, std::, <cmath>)
 SYMBOL(fmin, std::, <cmath>)
 SYMBOL(fmod, std::, <cmath>)

 SYMBOL(hexfloat, std::, <ios>)
 SYMBOL(holds_alternative, std::, <variant>)
 SYMBOL(hypot, std::, <cmath>)
 SYMBOL(identity, std::, <functional>)
 SYMBOL(ifstream, std::, <fstream>)
 SYMBOL(ignore, std::, <tuple>)
 SYMBOL(ilogb, std::, <cmath>)
 SYMBOL(imag, std::, <complex>)
+SYMBOL(imaxabs, std::, <cinttypes>)
+SYMBOL(imaxdiv, std::, <cinttypes>)
+SYMBOL(imaxdiv_t, std::, <cinttypes>)
 SYMBOL(in_place, std::, <utility>)
 SYMBOL(in_place_index, std::, <utility>)
 SYMBOL(in_place_index_t, std::, <utility>)
 SYMBOL(in_place_t, std::, <utility>)
 SYMBOL(in_place_type, std::, <utility>)
 SYMBOL(in_place_type_t, std::, <utility>)
 SYMBOL(includes, std::, <algorithm>)
 SYMBOL(inclusive_scan, std::, <numeric>)
 SYMBOL(independent_bits_engine, std::, <random>)
 SYMBOL(indirect_array, std::, <valarray>)
 SYMBOL(inner_product, std::, <numeric>)
 SYMBOL(inplace_merge, std::, <algorithm>)
 SYMBOL(input_iterator_tag, std::, <iterator>)
 SYMBOL(insert_iterator, std::, <iterator>)
 SYMBOL(inserter, std::, <iterator>)
 SYMBOL(integer_sequence, std::, <utility>)
 SYMBOL(integral_constant, std::, <type_traits>)
 SYMBOL(internal, std::, <ios>)
+SYMBOL(intmax_t, std::, <cstdint>)
+SYMBOL(intptr_t, std::, <cstdint>)
 SYMBOL(invalid_argument, std::, <stdexcept>)
 SYMBOL(invoke, std::, <functional>)
 SYMBOL(invoke_result, std::, <type_traits>)
 SYMBOL(invoke_result_t, std::, <type_traits>)
 SYMBOL(io_errc, std::, <ios>)
 SYMBOL(ios, std::, <ios>)
 SYMBOL(ios_base, std::, <ios>)
 SYMBOL(iostream, std::, <istream>)

 SYMBOL(isgreaterequal, std::, <cmath>)
 SYMBOL(isinf, std::, <cmath>)
 SYMBOL(isless, std::, <cmath>)
 SYMBOL(islessequal, std::, <cmath>)
 SYMBOL(islessgreater, std::, <cmath>)
 SYMBOL(isnan, std::, <cmath>)
 SYMBOL(isnormal, std::, <cmath>)
 SYMBOL(ispow2, std::, <bit>)
+SYMBOL(istream, std::, <istream>)
 SYMBOL(istream_iterator, std::, <iterator>)
 SYMBOL(istreambuf_iterator, std::, <iterator>)
 SYMBOL(istringstream, std::, <sstream>)
 SYMBOL(isunordered, std::, <cmath>)
 SYMBOL(iswalnum, std::, <cwctype>)
 SYMBOL(iswalpha, std::, <cwctype>)
 SYMBOL(iswblank, std::, <cwctype>)
 SYMBOL(iswcntrl, std::, <cwctype>)
 SYMBOL(iswctype, std::, <cwctype>)
 SYMBOL(iswdigit, std::, <cwctype>)
 SYMBOL(iswgraph, std::, <cwctype>)
 SYMBOL(iswlower, std::, <cwctype>)
 SYMBOL(iswprint, std::, <cwctype>)
 SYMBOL(iswpunct, std::, <cwctype>)
 SYMBOL(iswspace, std::, <cwctype>)
 SYMBOL(iswupper, std::, <cwctype>)
 SYMBOL(iswxdigit, std::, <cwctype>)
 SYMBOL(iter_swap, std::, <algorithm>)
 SYMBOL(iterator, std::, <iterator>)
 SYMBOL(iterator_traits, std::, <iterator>)
 SYMBOL(jmp_buf, std::, <csetjmp>)
 SYMBOL(kill_dependency, std::, <atomic>)
 SYMBOL(kilo, std::, <ratio>)
+SYMBOL(knuth_b, std::, <random>)
+SYMBOL(labs, std::, <cstdlib>)
 SYMBOL(launch, std::, <future>)
 SYMBOL(launder, std::, <new>)
 SYMBOL(lcm, std::, <numeric>)
 SYMBOL(lconv, std::, <clocale>)
 SYMBOL(ldexp, std::, <cmath>)
+SYMBOL(ldiv, std::, <cstdlib>)
+SYMBOL(ldiv_t, std::, <cstdlib>)
 SYMBOL(left, std::, <ios>)
 SYMBOL(length_error, std::, <stdexcept>)
 SYMBOL(less, std::, <functional>)
 SYMBOL(less_equal, std::, <functional>)
 SYMBOL(lexicographical_compare, std::, <algorithm>)
 SYMBOL(lexicographical_compare_3way, std::, <algorithm>)
 SYMBOL(lgamma, std::, <cmath>)
 SYMBOL(linear_congruential_engine, std::, <random>)
 SYMBOL(list, std::, <list>)
+SYMBOL(llabs, std::, <cstdlib>)
+SYMBOL(lldiv, std::, <cstdlib>)
+SYMBOL(lldiv_t, std::, <cstdlib>)
 SYMBOL(llrint, std::, <cmath>)
 SYMBOL(llround, std::, <cmath>)
 SYMBOL(locale, std::, <locale>)
 SYMBOL(localeconv, std::, <clocale>)
 SYMBOL(localtime, std::, <ctime>)
 SYMBOL(lock, std::, <mutex>)
 SYMBOL(lock_guard, std::, <mutex>)
 SYMBOL(log1p, std::, <cmath>)

 SYMBOL(messages_base, std::, <locale>)
 SYMBOL(messages_byname, std::, <locale>)
 SYMBOL(micro, std::, <ratio>)
 SYMBOL(milli, std::, <ratio>)
 SYMBOL(min, std::, <algorithm>)
 SYMBOL(min_element, std::, <algorithm>)
 SYMBOL(minmax, std::, <algorithm>)
 SYMBOL(minmax_element, std::, <algorithm>)
+SYMBOL(minstd_rand, std::, <random>)
+SYMBOL(minstd_rand0, std::, <random>)
 SYMBOL(minus, std::, <functional>)
 SYMBOL(mismatch, std::, <algorithm>)
 SYMBOL(mktime, std::, <ctime>)
 SYMBOL(modf, std::, <cmath>)
 SYMBOL(modulus, std::, <functional>)
 SYMBOL(money_base, std::, <locale>)
 SYMBOL(money_get, std::, <locale>)
 SYMBOL(money_put, std::, <locale>)
 SYMBOL(moneypunct, std::, <locale>)
 SYMBOL(moneypunct_byname, std::, <locale>)
 SYMBOL(monostate, std::, <variant>)
 SYMBOL(move_backward, std::, <algorithm>)
 SYMBOL(move_if_noexcept, std::, <utility>)
 SYMBOL(move_iterator, std::, <iterator>)
+SYMBOL(mt19937, std::, <random>)
+SYMBOL(mt19937_64, std::, <random>)
 SYMBOL(multimap, std::, <map>)
 SYMBOL(multiplies, std::, <functional>)
 SYMBOL(multiset, std::, <set>)
 SYMBOL(mutex, std::, <mutex>)
 SYMBOL(nan, std::, <cmath>)
 SYMBOL(nanf, std::, <cmath>)
 SYMBOL(nanl, std::, <cmath>)
 SYMBOL(nano, std::, <ratio>)

 SYMBOL(num_put, std::, <locale>)
 SYMBOL(numeric_limits, std::, <limits>)
 SYMBOL(numpunct, std::, <locale>)
 SYMBOL(numpunct_byname, std::, <locale>)
 SYMBOL(oct, std::, <ios>)
 SYMBOL(ofstream, std::, <fstream>)
 SYMBOL(once_flag, std::, <mutex>)
 SYMBOL(optional, std::, <optional>)
+SYMBOL(ostream, std::, <ostream>)
 SYMBOL(ostream_iterator, std::, <iterator>)
 SYMBOL(ostreambuf_iterator, std::, <iterator>)
 SYMBOL(ostringstream, std::, <sstream>)
 SYMBOL(osyncstream, std::, <syncstream>)
 SYMBOL(out_of_range, std::, <stdexcept>)
 SYMBOL(output_iterator_tag, std::, <iterator>)
 SYMBOL(overflow_error, std::, <stdexcept>)
 SYMBOL(owner_less, std::, <memory>)

 SYMBOL(raise, std::, <csignal>)
 SYMBOL(rand, std::, <cstdlib>)
 SYMBOL(random_access_iterator_tag, std::, <iterator>)
 SYMBOL(random_device, std::, <random>)
 SYMBOL(random_shuffle, std::, <algorithm>)
 SYMBOL(range_error, std::, <stdexcept>)
 SYMBOL(rank, std::, <type_traits>)
 SYMBOL(rank_v, std::, <type_traits>)
+SYMBOL(ranlux24, std::, <random>)
+SYMBOL(ranlux24_base, std::, <random>)
+SYMBOL(ranlux48, std::, <random>)
+SYMBOL(ranlux48_base, std::, <random>)
 SYMBOL(ratio, std::, <ratio>)
 SYMBOL(ratio_add, std::, <ratio>)
 SYMBOL(ratio_divide, std::, <ratio>)
 SYMBOL(ratio_equal, std::, <ratio>)
 SYMBOL(ratio_equal_v, std::, <ratio>)
 SYMBOL(ratio_greater, std::, <ratio>)
 SYMBOL(ratio_greater_equal, std::, <ratio>)
 SYMBOL(ratio_greater_equal_v, std::, <ratio>)

 SYMBOL(type_index, std::, <typeindex>)
 SYMBOL(type_info, std::, <typeinfo>)
 SYMBOL(u16streampos, std::, <ios>)
 SYMBOL(u16string, std::, <string>)
 SYMBOL(u16string_view, std::, <string_view>)
 SYMBOL(u32streampos, std::, <ios>)
 SYMBOL(u32string, std::, <string>)
 SYMBOL(u32string_view, std::, <string_view>)
+SYMBOL(uintmax_t, std::, <cstdint>)
+SYMBOL(uintptr_t, std::, <cstdint>)
 SYMBOL(uncaught_exceptions, std::, <exception>)
 SYMBOL(undeclare_no_pointers, std::, <memory>)
 SYMBOL(undeclare_reachable, std::, <memory>)
 SYMBOL(underflow_error, std::, <stdexcept>)
 SYMBOL(underlying_type, std::, <type_traits>)
 SYMBOL(underlying_type_t, std::, <type_traits>)
 SYMBOL(ungetc, std::, <cstdio>)
 SYMBOL(ungetwc, std::, <cwchar>)

 SYMBOL(wcstoul, std::, <cwchar>)
 SYMBOL(wcstoull, std::, <cwchar>)
 SYMBOL(wcstoumax, std::, <cinttypes>)
 SYMBOL(wcsub_match, std::, <regex>)
 SYMBOL(wcsxfrm, std::, <cwchar>)
 SYMBOL(wctob, std::, <cwchar>)
 SYMBOL(wctomb, std::, <cstdlib>)
 SYMBOL(wctrans, std::, <cwctype>)
+SYMBOL(wctrans_t, std::, <cwctype>)
 SYMBOL(wctype, std::, <cwctype>)
+SYMBOL(wctype_t, std::, <cwctype>)
 SYMBOL(weak_equal, std::, <compare>)
 SYMBOL(weak_equality, std::, <compare>)
 SYMBOL(weak_order, std::, <compare>)
 SYMBOL(weak_ordering, std::, <compare>)
 SYMBOL(weak_ptr, std::, <memory>)
 SYMBOL(weibull_distribution, std::, <random>)
+SYMBOL(wfilebuf, std::, <streambuf>)
 SYMBOL(wfstream, std::, <fstream>)
 SYMBOL(wifstream, std::, <fstream>)
 SYMBOL(wios, std::, <ios>)
 SYMBOL(wiostream, std::, <istream>)
+SYMBOL(wistream, std::, <istream>)
 SYMBOL(wistringstream, std::, <sstream>)
 SYMBOL(wmemchr, std::, <cwchar>)
 SYMBOL(wmemcmp, std::, <cwchar>)
 SYMBOL(wmemcpy, std::, <cwchar>)
 SYMBOL(wmemmove, std::, <cwchar>)
 SYMBOL(wmemset, std::, <cwchar>)
 SYMBOL(wofstream, std::, <fstream>)
+SYMBOL(wostream, std::, <ostream>)
 SYMBOL(wostringstream, std::, <sstream>)
 SYMBOL(wosyncstream, std::, <syncstream>)
 SYMBOL(wprintf, std::, <cwchar>)
 SYMBOL(wregex, std::, <regex>)
 SYMBOL(ws, std::, <istream>)
 SYMBOL(wscanf, std::, <cwchar>)
 SYMBOL(wsmatch, std::, <regex>)
 SYMBOL(wsregex_iterator, std::, <regex>)

clangd/include-mapping/gen_std.py

3. Unzip the zip file from step 2 to directory </cppreference>, you should
      get a "reference" directory in </cppreference>
   4. Run the command:
        gen_std.py -cppreference </cppreference/reference> > StdSymbolMap.inc
 """
 
 from bs4 import BeautifulSoup
 
 import argparse
+import collections
 import datetime
+import multiprocessing
 import os
+import re
+import signal
 import sys
 
 STDGEN_CODE_PREFIX = """\
 //===-- gen_std.py generated file -------------------------------*- C++ -*-===//
 //
 // Used to build a lookup table (qualified names => include headers) for C++
 // Standard Library symbols.
 //
 // Automatically generated file, DO NOT EDIT!
 //
 // Generated from cppreference offline HTML book (modified on %s).
 //===----------------------------------------------------------------------===//
 """
 
-def ParseSymbolPage(symbol_page_html):
+def HasClass(tag, *classes):
+  for c in tag.get('class', []):
+    if c in classes:
+      return True
+  return False
+
+def ParseSymbolPage(symbol_page_html, symbol_name):
   """Parse symbol page and retrieve the include header defined in this page.
   The symbol page provides header for the symbol, specifically in
   "Defined in header <header>" section. An example:
 
   <tr class="t-dsc-header">
     <td colspan="2"> <div>Defined in header <code>&lt;ratio&gt;</code> </div>
   </td></tr>
 
   Returns a list of headers.
   """
-  headers = []
+  headers = set()
+  all_headers = set()
 
   soup = BeautifulSoup(symbol_page_html, "html.parser")
-  #  "Defined in header " are defined in <tr class="t-dsc-header"> or
-  #  <tr class="t-dcl-header">.
-  for header_tr in soup.select('tr.t-dcl-header,tr.t-dsc-header'):
-    if "Defined in header " in header_tr.text:
+  # Rows in table are like:
+  #   Defined in header <foo>      .t-dsc-header
+  #   Defined in header <bar>      .t-dsc-header
+  #   decl1                        .t-dcl
+  #   Defined in header <baz>      .t-dsc-header
+  #   decl2                        .t-dcl
+  for table in soup.select('table.t-dcl-begin, table.t-dsc-begin'):
+    current_headers = []
+    was_decl = False
+    for row in table.select('tr'):
+      if HasClass(row, 't-dcl', 't-dsc'):
+        was_decl = True
+        # Declaration is in the first cell.
+        text = row.find('td').text
+        # Decl may not be for the symbol name we're looking for.
+        if not re.search("\\b%s\\b" % symbol_name, text):
+          continue
+        headers.update(current_headers)
+      elif HasClass(row, 't-dsc-header'):
+        if was_decl:
+          current_headers = []
+        was_decl = False
+        # There are also .t-dsc-header for "defined in namespace".
+        if not "Defined in header " in row.text:
+          continue
         # The interesting header content (e.g. <cstdlib>) is wrapped in <code>.
-      for header_code in header_tr.find_all("code"):
-        headers.append(header_code.text)
-  return headers
+        for header_code in row.find_all("code"):
+          current_headers.append(header_code.text)
+          all_headers.add(header_code.text)
+  # If the symbol was never named, consider all named headers.
+  return headers or all_headers
 
 
 def ParseIndexPage(index_page_html):
   """Parse index page.
   The index page lists all std symbols and hrefs to their detailed pages
   (which contain the defined header). An example:
 
   <a href="abs.html" title="abs"><tt>abs()</tt></a> (int) <br>
def __init__(self, name, namespace, headers):
     # unqualifed symbol name, e.g. "move"
     self.name = name
     # namespace of the symbol (with trailing "::"), e.g. "std::"
     self.namespace = namespace
     # a list of corresponding headers
     self.headers = headers
 
 
-def GetSymbols(root_dir, index_page_name, namespace):
+def ReadSymbolPage(path, name):
+  with open(path) as f:
+    return ParseSymbolPage(f.read(), name)
+
+
+def GetSymbols(pool, root_dir, index_page_name, namespace):
   """Get all symbols listed in the index page. All symbols should be in the
   given namespace.
 
   Returns a list of Symbols.
   """
 
   # Workflow steps:
   #   1. Parse index page which lists all symbols to get symbol
   #      name (unqualified name) and its href link to the symbol page which
   #      contains the defined header.
   #   2. Parse the symbol page to get the defined header.
   index_page_path = os.path.join(root_dir, index_page_name)
-  symbols = []
   with open(index_page_path, "r") as f:
-    # A map from symbol name to a set of headers.
-    symbol_headers = {}
+    # Read each symbol page in parallel.
+    results = [] # (symbol_name, promise of [header...])
     for symbol_name, symbol_page_path in ParseIndexPage(f.read()):
-      with open(os.path.join(root_dir, symbol_page_path), "r") as f:
-        headers = ParseSymbolPage(f.read())
-      if not headers:
-        sys.stderr.write("No header found for symbol %s at %s\n" % (symbol_name,
-          symbol_page_path))
-        continue
-
-      if symbol_name not in symbol_headers:
-        symbol_headers[symbol_name] = set()
-      symbol_headers[symbol_name].update(headers)
+      path = os.path.join(root_dir, symbol_page_path)
+      results.append((symbol_name,
+                      pool.apply_async(ReadSymbolPage, (path, symbol_name))))
+
+    # Build map from symbol name to a set of headers.
+    symbol_headers = collections.defaultdict(set)
+    for symbol_name, lazy_headers in results:
+      symbol_headers[symbol_name].update(lazy_headers.get())
 
-    for name, headers in sorted(symbol_headers.items(), key=lambda t : t[0]):
-      symbols.append(Symbol(name, namespace, list(headers)))
+  symbols = []
   return symbols
 
 
 def ParseArg():
   parser = argparse.ArgumentParser(description='Generate StdGen file')
   parser.add_argument('-cppreference', metavar='PATH',
                       default='',
                       help='path to the cppreference offline HTML directory',
parse_pages =  [
     (symbol_index_root, "chrono.html", "std::chrono::"),
     (symbol_index_root, "filesystem.html", "std::filesystem::"),
     (symbol_index_root, "pmr.html", "std::pmr::"),
     (symbol_index_root, "regex_constants.html", "std::regex_constants::"),
     (symbol_index_root, "this_thread.html", "std::this_thread::"),
   ]
 
   symbols = []
+  # Run many workers to process individual symbol pages under the symbol index.
+  # Don't allow workers to capture Ctrl-C.
+  pool = multiprocessing.Pool(
+      initializer=lambda: signal.signal(signal.SIGINT, signal.SIG_IGN))
+  try:
     for root_dir, page_name, namespace in parse_pages:
-    symbols.extend(GetSymbols(root_dir, page_name, namespace))
+      symbols.extend(GetSymbols(pool, root_dir, page_name, namespace))
+  finally:
+    pool.terminate()
+    pool.join()
 
   # We don't have version information from the unzipped offline HTML files.
   # so we use the modified time of the symbol_index.html as the version.
   index_page_path = os.path.join(cpp_root, "symbol_index.html")
   cppreference_modified_date = datetime.datetime.fromtimestamp(
     os.stat(index_page_path).st_mtime).strftime('%Y-%m-%d')
   print STDGEN_CODE_PREFIX % cppreference_modified_date
   for symbol in symbols:
-    if len(symbol.headers) > 1:
-      # FIXME: support symbols with multiple headers (e.g. std::move).
-      continue
+    if len(symbol.headers) == 1:
       # SYMBOL(unqualified_name, namespace, header)
       print "SYMBOL(%s, %s, %s)" % (symbol.name, symbol.namespace,
                                     symbol.headers[0])
+    elif len(symbol.headers) == 0:
+      sys.stderr.write("No header found for symbol %s\n" % symbol.name)
+    else:
+      # FIXME: support symbols with multiple headers (e.g. std::move).
+      sys.stderr.write("Ambiguous header for symbol %s: %s\n" % (
+          symbol.name, ', '.join(symbol.headers)))
 
 
 if __name__ == '__main__':
   main()

clangd/include-mapping/test.py

def testParseSymbolPage_SingleHeader(self):
     html = """
  <table class="t-dcl-begin"><tbody>
   <tr class="t-dsc-header">
   <td> <div>Defined in header <code><a href="cmath.html" title="cmath">&lt;cmath&gt;</a></code>
    </div></td>
   <td></td>
   <td></td>
   </tr>
+  <tr class="t-dcl">
+    <td>void foo()</td>
+    <td>this is matched</td>
+  </tr>
 </tbody></table>
 """
-    self.assertEqual(ParseSymbolPage(html), ['<cmath>'])
+    self.assertEqual(ParseSymbolPage(html, 'foo'), set(['<cmath>']))
 
 
   def testParseSymbolPage_MulHeaders(self):
     #  Defined in header <cstddef>
     #  Defined in header <cstdio>
     #  Defined in header <cstdlib>
     html = """
 <table class="t-dcl-begin"><tbody>
   <tr class="t-dsc-header">
     <td> <div>Defined in header <code><a href="cstddef.html" title="cstddef">&lt;cstddef&gt;</a></code>
      </div></td>
      <td></td>
     <td></td>
   </tr>
+  <tr class="t-dcl">
+    <td>void bar()</td>
+    <td>this mentions foo, but isn't matched</td>
+  </tr>
   <tr class="t-dsc-header">
     <td> <div>Defined in header <code><a href="cstdio.html" title="cstdio">&lt;cstdio&gt;</a></code>
      </div></td>
     <td></td>
     <td></td>
   </tr>
   <tr class="t-dsc-header">
     <td> <div>Defined in header <code><a href=".cstdlib.html" title="ccstdlib">&lt;cstdlib&gt;</a></code>
      </div></td>
     <td></td>
     <td></td>
   </tr>
+  <tr class="t-dcl">
+    <td>void foo()</td>
+    <td>this is matched</td>
+  </tr>
 </tbody></table>
 """
-    self.assertEqual(ParseSymbolPage(html),
-                    ['<cstddef>', '<cstdio>', '<cstdlib>'])
+    self.assertEqual(ParseSymbolPage(html, "foo"),
+                     set(['<cstdio>', '<cstdlib>']))
 
 
   def testParseSymbolPage_MulHeadersInSameDiv(self):
     # Multile <code> blocks in a Div.
     # Defined in header <algorithm>
     # Defined in header <utility>
     html = """
+<table class="t-dcl-begin"><tbody>
 <tr class="t-dsc-header">
 <td><div>
      Defined in header <code><a href="../header/algorithm.html" title="cpp/header/algorithm">&lt;algorithm&gt;</a></code><br>
      Defined in header <code><a href="../header/utility.html" title="cpp/header/utility">&lt;utility&gt;</a></code>
 </div></td>
 <td></td>
 </tr>
+<tr class="t-dcl">
+  <td>void foo()</td>
+  <td>this is matched</td>
+</tr>
+</tbody></table>
 """
-    self.assertEqual(ParseSymbolPage(html), ['<algorithm>', '<utility>'])
+    self.assertEqual(ParseSymbolPage(html, "foo"),
+                     set(['<algorithm>', '<utility>']))
 
 
 if __name__ == '__main__':
   unittest.main()

clangd/index/Serialization.cpp

Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
   Sym.SymInfo.Kind = static_cast<index::SymbolKind>(Data.consume8());
   Sym.SymInfo.Lang = static_cast<index::SymbolLanguage>(Data.consume8());
   Sym.Name = Data.consumeString(Strings);
   Sym.Scope = Data.consumeString(Strings);
   Sym.TemplateSpecializationArgs = Data.consumeString(Strings);
   Sym.Definition = readLocation(Data, Strings);
   Sym.CanonicalDeclaration = readLocation(Data, Strings);
   Sym.References = Data.consumeVar();
-  Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consumeVar());
-  Sym.Origin = static_cast<SymbolOrigin>(Data.consumeVar());
+  Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consume8());
+  Sym.Origin = static_cast<SymbolOrigin>(Data.consume8());
   Sym.Signature = Data.consumeString(Strings);
   Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
   Sym.Documentation = Data.consumeString(Strings);
   Sym.ReturnType = Data.consumeString(Strings);
   Sym.Type = Data.consumeString(Strings);
   Sym.IncludeHeaders.resize(Data.consumeVar());
   for (auto &I : Sym.IncludeHeaders) {
     I.IncludeHeader = Data.consumeString(Strings);

clangd/test/CMakeLists.txt

 set(CLANGD_TEST_DEPS
   clangd
   ClangdTests
   # No tests for these, but we should still make sure they build.
   clangd-indexer
   dexp
   )
 
 if(CLANGD_BUILD_XPC)
   list(APPEND CLANGD_TEST_DEPS clangd-xpc-test-client)
+  list(APPEND CLANGD_TEST_DEPS ClangdXpcUnitTests)
 endif()
 
 foreach(dep FileCheck count not)
   if(TARGET ${dep})
     list(APPEND CLANGD_TEST_DEPS ${dep})
   endif()
 endforeach()
 
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.cfg)
 
 add_lit_testsuite(check-clangd "Running the Clangd regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}/../unittests;${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${CLANGD_TEST_DEPS})

clangd/unittests/CMakeLists.txt

if(CLANG_BUILT_STANDALONE)
   endif()
 endif()
 
 add_custom_target(ClangdUnitTests)
 add_unittest(ClangdUnitTests ClangdTests
   Annotations.cpp
   BackgroundIndexTests.cpp
   CancellationTests.cpp
+  CanonicalIncludesTests.cpp
   ClangdTests.cpp
   ClangdUnitTests.cpp
   CodeCompleteTests.cpp
   CodeCompletionStringsTests.cpp
   ContextTests.cpp
   DexTests.cpp
   DiagnosticsTests.cpp
   DraftStoreTests.cpp

clangd/unittests/CanonicalIncludesTests.cpp

+//===-- CanonicalIncludesTests.cpp - --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/CanonicalIncludes.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TEST(CanonicalIncludesTest, CXXStandardLibrary) {
+  CanonicalIncludes CI;
+  addSystemHeadersMapping(&CI);
+
+  // Usual standard library symbols are mapped correctly.
+  EXPECT_EQ("<vector>", CI.mapHeader("path/vector.h", "std::vector"));
+  // std::move is ambiguous, currently mapped only based on path
+  EXPECT_EQ("<utility>", CI.mapHeader("libstdc++/bits/move.h", "std::move"));
+  EXPECT_EQ("path/utility.h", CI.mapHeader("path/utility.h", "std::move"));
+  // Unknown std symbols aren't mapped.
+  EXPECT_EQ("foo/bar.h", CI.mapHeader("foo/bar.h", "std::notathing"));
+  // iosfwd declares some symbols it doesn't own.
+  EXPECT_EQ("<ostream>", CI.mapHeader("iosfwd", "std::ostream"));
+  // And (for now) we assume it owns the others.
+  EXPECT_EQ("<iosfwd>", CI.mapHeader("iosfwd", "std::notwathing"));
+}
+
+TEST(CanonicalIncludesTest, PathMapping) {
+  // As used for IWYU pragmas.
+  CanonicalIncludes CI;
+  CI.addMapping("foo/bar", "<baz>");
+
+  EXPECT_EQ("<baz>", CI.mapHeader("foo/bar", "some::symbol"));
+  EXPECT_EQ("bar/bar", CI.mapHeader("bar/bar", "some::symbol"));
+}
+
+TEST(CanonicalIncludesTest, SymbolMapping) {
+  // As used for standard library.
+  CanonicalIncludes CI;
+  CI.addSymbolMapping("some::symbol", "<baz>");
+
+  EXPECT_EQ("<baz>", CI.mapHeader("foo/bar", "some::symbol"));
+  EXPECT_EQ("foo/bar", CI.mapHeader("foo/bar", "other::symbol"));
+}
+
+TEST(CanonicalIncludesTest, Precedence) {
+  CanonicalIncludes CI;
+  CI.addMapping("some/path", "<path>");
+  CI.addSymbolMapping("some::symbol", "<symbol>");
+
+  // Symbol mapping beats path mapping.
+  EXPECT_EQ("<symbol>", CI.mapHeader("some/path", "some::symbol"));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang

clangd/unittests/SerializationTests.cpp

 CanonicalDeclaration:
   FileURI:        file:///path/foo.h
   Start:
     Line: 1
     Column: 0
   End:
     Line: 1
     Column: 1
-Origin:    4
-Flags:    1
+Origin:    128
+Flags:    129
 Documentation:    'Foo doc'
 ReturnType:    'int'
 IncludeHeaders:
   - Header:    'include1'
     References:    7
   - Header:    'include2'
     References:    3
 ...
TEST(SerializationTest, YAMLConversions) {
   auto Sym2 = *ParsedYAML->Symbols->find(
       cantFail(SymbolID::fromStr("057557CEBF6E6B2E")));
 
   EXPECT_THAT(Sym1, QName("clang::Foo1"));
   EXPECT_EQ(Sym1.Signature, "");
   EXPECT_EQ(Sym1.Documentation, "Foo doc");
   EXPECT_EQ(Sym1.ReturnType, "int");
   EXPECT_EQ(StringRef(Sym1.CanonicalDeclaration.FileURI), "file:///path/foo.h");
-  EXPECT_EQ(Sym1.Origin, SymbolOrigin::Static);
+  EXPECT_EQ(Sym1.Origin, static_cast<SymbolOrigin>(1 << 7));
+  EXPECT_EQ(static_cast<uint8_t>(Sym1.Flags), 129);
   EXPECT_TRUE(Sym1.Flags & Symbol::IndexedForCodeCompletion);
   EXPECT_FALSE(Sym1.Flags & Symbol::Deprecated);
   EXPECT_THAT(Sym1.IncludeHeaders,
               UnorderedElementsAre(IncludeHeaderWithRef("include1", 7u),
                                    IncludeHeaderWithRef("include2", 3u)));
 
   EXPECT_THAT(Sym2, QName("clang::Foo2"));
   EXPECT_EQ(Sym2.Signature, "-sig");

clangd/unittests/SymbolCollectorTests.cpp

TEST_F(SymbolCollectorTest, IncludeHeaderSameAsFileURI) {
   CollectorOpts.CollectIncludePath = true;
   runSymbolCollector("#pragma once\nclass Foo {};", /*Main=*/"");
   EXPECT_THAT(Symbols, UnorderedElementsAre(
                            AllOf(QName("Foo"), DeclURI(TestHeaderURI))));
   EXPECT_THAT(Symbols.begin()->IncludeHeaders,
               UnorderedElementsAre(IncludeHeaderWithRef(TestHeaderURI, 1u)));
 }
 
-#ifndef _WIN32
 TEST_F(SymbolCollectorTest, CanonicalSTLHeader) {
   CollectorOpts.CollectIncludePath = true;
   CanonicalIncludes Includes;
   addSystemHeadersMapping(&Includes);
   CollectorOpts.Includes = &Includes;
-  // bits/basic_string.h$ should be mapped to <string>
-  TestHeaderName = "/nasty/bits/basic_string.h";
-  TestFileName = "/nasty/bits/basic_string.cpp";
-  TestHeaderURI = URI::create(TestHeaderName).toString();
-  runSymbolCollector("class string {};", /*Main=*/"");
-  EXPECT_THAT(Symbols, UnorderedElementsAre(AllOf(QName("string"),
-                                                  DeclURI(TestHeaderURI),
-                                                  IncludeHeader("<string>"))));
-}
-#endif
-
-TEST_F(SymbolCollectorTest, STLiosfwd) {
-  CollectorOpts.CollectIncludePath = true;
-  CanonicalIncludes Includes;
-  addSystemHeadersMapping(&Includes);
-  CollectorOpts.Includes = &Includes;
-  // Symbols from <iosfwd> should be mapped individually.
-  TestHeaderName = testPath("iosfwd");
-  TestFileName = testPath("iosfwd.cpp");
-  std::string Header = R"(
-    namespace std {
-      class no_map {};
-      class ios {};
-      class ostream {};
-      class filebuf {};
-    } // namespace std
-  )";
-  runSymbolCollector(Header, /*Main=*/"");
+  runSymbolCollector("namespace std { class string {}; }", /*Main=*/"");
   EXPECT_THAT(Symbols,
-              UnorderedElementsAre(
-                  QName("std"),
-                  AllOf(QName("std::no_map"), IncludeHeader("<iosfwd>")),
-                  AllOf(QName("std::ios"), IncludeHeader("<ios>")),
-                  AllOf(QName("std::ostream"), IncludeHeader("<ostream>")),
-                  AllOf(QName("std::filebuf"), IncludeHeader("<fstream>"))));
+              Contains(AllOf(QName("std::string"), DeclURI(TestHeaderURI),
+                             IncludeHeader("<string>"))));
 }
 
 TEST_F(SymbolCollectorTest, IWYUPragma) {
   CollectorOpts.CollectIncludePath = true;
   CanonicalIncludes Includes;
   PragmaHandler = collectIWYUHeaderMaps(&Includes);
   CollectorOpts.Includes = &Includes;
   const std::string Header = R"(

clangd/unittests/xpc/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   support
   )
 
 get_filename_component(CLANGD_SOURCE_DIR
   ${CMAKE_CURRENT_SOURCE_DIR}/../../clangd REALPATH)
 include_directories(
   ${CLANGD_SOURCE_DIR}
   )
 
-add_extra_unittest(ClangdXpcTests
+add_custom_target(ClangdXpcUnitTests)
+add_unittest(ClangdXpcUnitTests ClangdXpcTests
   ConversionTests.cpp
   )
 
 target_link_libraries(ClangdXpcTests
   PRIVATE
   clangdXpcJsonConversions
   clangDaemon
   LLVMSupport
   LLVMTestingSupport
   )