[clangd] Print the ignored variant symbols

[hokein]

Summary

so that we can better track those symbols.

[sammccall]

clang-tools-extra/clangd/include-mapping/gen_std.py

119

this is a significantly more complicated signature - do we need the extra precision in the log message vs e.g. Ignored variant of symbol abs from path/to/complex.html?

[hokein]

clang-tools-extra/clangd/include-mapping/gen_std.py

119

I think the current information is sufficient (Ignore variant of symbol abs(complex)), we are more caring about the entry name symbol_name (caption) (it is easy for us to find it in the symbol index page).

clang-tools-extra/clangd/include-mapping/gen_std.py

 def ParseIndexPage(index_page_html):
   """Parse index page.
   The index page lists all std symbols and hrefs to their detailed pages
   (which contain the defined header). An example:
 
   <a href="abs.html" title="abs"><tt>abs()</tt></a> (int) <br>
   <a href="acos.html" title="acos"><tt>acos()</tt></a> <br>
 
-  Returns a list of tuple (symbol_name, relative_path_to_symbol_page, variant).
+  Returns a tuple
+    ( a list of tuple (symbol_name, relative_path_to_symbol_page),
+      a list of tuple (variant_symbol_name, caption_text) )
   """
   symbols = []
+  variant_symbols = []
   soup = BeautifulSoup(index_page_html, "html.parser")
   for symbol_href in soup.select("a[title]"):
     # Ignore annotated symbols like "acos<>() (std::complex)".
     # These tend to be overloads, and we the primary is more useful.
     # This accidentally accepts begin/end despite the (iterator) caption: the
     # (since C++11) note is first. They are good symbols, so the bug is unfixed.
     caption = symbol_href.next_sibling
     variant = isinstance(caption, NavigableString) and "(" in caption
     symbol_tt = symbol_href.find("tt")
     if symbol_tt:
-      symbols.append((symbol_tt.text.rstrip("<>()"), # strip any trailing <>()
-                      symbol_href["href"], variant))
-  return symbols
+      symbol_name = symbol_tt.text.rstrip("<>()") # strip any trailing <>()
+      if variant:
+        variant_symbols.append((symbol_name, caption))
+      else:
+        symbols.append((symbol_name, symbol_href["href"]))
+  return (symbols, variant_symbols)
 
 class Symbol:
 
   def __init__(self, name, namespace, headers):
     # unqualifed symbol name, e.g. "move"
     self.name = name
     # namespace of the symbol (with trailing "::"), e.g. "std::"
     self.namespace = namespace
def GetSymbols(pool, root_dir, index_page_name, namespace):
   #   1. Parse index page which lists all symbols to get symbol
   #      name (unqualified name) and its href link to the symbol page which
   #      contains the defined header.
   #   2. Parse the symbol page to get the defined header.
   index_page_path = os.path.join(root_dir, index_page_name)
   with open(index_page_path, "r") as f:
     # Read each symbol page in parallel.
     results = [] # (symbol_name, promise of [header...])
-    for symbol_name, symbol_page_path, variant in ParseIndexPage(f.read()):
-      # Variant symbols (e.g. the std::locale version of isalpha) add ambiguity.
-      # FIXME: use these as a fallback rather than ignoring entirely.
-      if variant:
-        continue
+    (symbols, variant_symbols) = ParseIndexPage(f.read())
+    for symbol_name, symbol_page_path in symbols:
       path = os.path.join(root_dir, symbol_page_path)
       results.append((symbol_name,
                       pool.apply_async(ReadSymbolPage, (path, symbol_name))))
 
     # Build map from symbol name to a set of headers.
     symbol_headers = collections.defaultdict(set)
     for symbol_name, lazy_headers in results:
       symbol_headers[symbol_name].update(lazy_headers.get())
 
+    for variant_symbol_name, caption in variant_symbols:
+      if variant_symbol_name not in symbol_headers:
+        sys.stderr.write("Ignore variant symbol %s%s\n" % (variant_symbol_name,
+             caption))
+
   symbols = []
   for name, headers in sorted(symbol_headers.items(), key=lambda t : t[0]):
     symbols.append(Symbol(name, namespace, list(headers)))
   return symbols
 
 
 def ParseArg():
   parser = argparse.ArgumentParser(description='Generate StdGen file')

clang-tools-extra/clangd/include-mapping/test.py

 from gen_std import ParseSymbolPage, ParseIndexPage
 
 import unittest
 
 class TestStdGen(unittest.TestCase):
 
   def testParseIndexPage(self):
     html = """
- <a href="abs.html" title="abs"><tt>abs()</tt></a> (int) <br>
- <a href="complex/abs.html" title="abs"><tt>abs&lt;&gt;()</tt></a> (std::complex) <br>
+ <a href="abs.html" title="abs"><tt>abs()</tt></a>(int)<br>
+ <a href="complex/abs.html" title="abs"><tt>abs&lt;&gt;()</tt></a>(std::complex)<br>
  <a href="acos.html" title="acos"><tt>acos()</tt></a> <br>
  <a href="acosh.html" title="acosh"><tt>acosh()</tt></a> <span class="t-mark-rev">(since C++11)</span> <br>
  <a href="as_bytes.html" title="as bytes"><tt>as_bytes&lt;&gt;()</tt></a> <span class="t-mark-rev t-since-cxx20">(since C++20)</span> <br>
  """
 
-    actual = ParseIndexPage(html)
-    expected = [
-      ("abs", "abs.html", True),
-      ("abs", "complex/abs.html", True),
-      ("acos", "acos.html", False),
-      ("acosh", "acosh.html", False),
-      ("as_bytes", "as_bytes.html", False),
+    (actual, variant) = ParseIndexPage(html)
+    expected_actual = [
+      ("acos", "acos.html"),
+      ("acosh", "acosh.html"),
+      ("as_bytes", "as_bytes.html"),
     ]
-    self.assertEqual(len(actual), len(expected))
+    expected_variant = [
+      ("abs", "(int)"),
+      ("abs", "(std::complex)")
+    ]
+    self.assertEqual(len(actual), len(expected_actual))
     for i in range(0, len(actual)):
-      self.assertEqual(expected[i][0], actual[i][0])
-      self.assertTrue(actual[i][1].endswith(expected[i][1]))
-      self.assertEqual(expected[i][2], actual[i][2])
+      self.assertEqual(expected_actual[i][0], actual[i][0])
+      self.assertTrue(actual[i][1].endswith(expected_actual[i][1]))
+    self.assertEqual(len(variant), len(expected_variant))
+    for i in range(0, len(variant)):
+      self.assertEqual(expected_variant[i], variant[i])
 
 
   def testParseSymbolPage_SingleHeader(self):
     # Defined in header <cmath>
     html = """
  <table class="t-dcl-begin"><tbody>
   <tr class="t-dsc-header">
   <td> <div>Defined in header <code><a href="cmath.html" title="cmath">&lt;cmath&gt;</a></code>