[libc++] Move the GDB pretty printer tests to the DSL

[ldionne]

Summary

Also, enable them whenever we detect that gdb is available. Previously,
these tests would basically never run because they relied on a CMake
configuration option that defaulted to OFF.

[ldionne]

@saugustine I'm seeing failures in the tests right now. Could you please kindly take a look?

Starting with this commit, I'd like us to run these tests all the time, otherwise it's no use having dead code in the repo.

[saugustine]

The reason this hasn't been run by default is due to python versioning issues with gdb, the system it runs on, and the tests themselves. There are certain cross-platform issues as well.

What failures are you seeing?

[ldionne]

Mark GDB tests as unsupported when localization is not supported

[ldionne]

In D91434#2394281, @saugustine wrote:

The reason this hasn't been run by default is due to python versioning issues with gdb, the system it runs on, and the tests themselves. There are certain cross-platform issues as well.

We should strive to detect whether we can run the tests automatically instead, which is what this patch is trying to do. Otherwise, we just end up with code that is never run, which is bad because it eventually rots. What are the requirements of the GDB tests?

What failures are you seeing?

I was seeing some failures locally in the CI Docker image, but I'm not seeing them anymore and the bots seem to be fine, so nevermind!

libcxx/CMakeLists.txt

option(LIBCXX_ENABLE_LOCALIZATION
    will be completely unusable, and other parts may be only partly available." ON)
 option(LIBCXX_ENABLE_VENDOR_AVAILABILITY_ANNOTATIONS
   "Whether to turn on vendor availability annotations on declarations that depend
    on definitions in a shared library. By default, we assume that we're not building
    libc++ for any specific vendor, and we disable those annotations. Vendors wishing
    to provide compile-time errors when using features unavailable on some version of
    the shared library they shipped should turn this on and see `include/__availability`
    for more details." OFF)
-option(LIBCXX_TEST_GDB_PRETTY_PRINTERS "Test gdb pretty printers." OFF)
 set(LIBCXX_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/configs/legacy.cfg.in" CACHE STRING
     "The Lit testing configuration to use when running the tests.")
 set(LIBCXX_TEST_PARAMS "" CACHE STRING
     "A list of parameters to run the Lit test suite with.")
 
 # Benchmark options -----------------------------------------------------------
 option(LIBCXX_INCLUDE_BENCHMARKS "Build the libc++ benchmarks and their dependencies" ON)
 

libcxx/test/CMakeLists.txt

set(LIBCXX_EXECUTOR "${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../utils/run.py" CACHE STRING
     "Executor to use when running tests.")
 
 set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!")
 
 if (NOT DEFINED LIBCXX_TEST_DEPS)
   message(FATAL_ERROR "Expected LIBCXX_TEST_DEPS to be defined")
 endif()
 
-# Turn this on by default when the pretty printers are python3
-# compatible.
-if(LIBCXX_TEST_GDB_PRETTY_PRINTERS)
-  find_program(LIBCXX_GDB gdb)
-  if (LIBCXX_GDB)
-    set(LIBCXX_GDB "${LIBCXX_GDB}")
-    message(STATUS "gdb found: ${LIBCXX_GDB}")
-  else()
-    message(STATUS "gdb not found. Disabling dependent tests.")
-  endif()
-endif()
-
 if (LIBCXX_INCLUDE_TESTS)
   include(AddLLVM) # for configure_lit_site_cfg and add_lit_target
 
   configure_lit_site_cfg(
     "${LIBCXX_TEST_CONFIG}"
     ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
     MAIN_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py")
 

libcxx/test/configs/legacy.cfg.in

 config.llvm_unwinder            = @LIBCXXABI_USE_LLVM_UNWINDER@
 config.builtins_library         = "@LIBCXX_BUILTINS_LIBRARY@"
 config.has_libatomic            = @LIBCXX_HAS_ATOMIC_LIB@
 config.debug_build              = @LIBCXX_DEBUG_BUILD@
 config.libcxxabi_shared         = @LIBCXX_LINK_TESTS_WITH_SHARED_LIBCXXABI@
 config.cxx_ext_threads          = @LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY@
 config.pstl_src_root            = "@ParallelSTL_SOURCE_DIR@" if @LIBCXX_ENABLE_PARALLEL_ALGORITHMS@ else None
 config.pstl_obj_root            = "@ParallelSTL_BINARY_DIR@" if @LIBCXX_ENABLE_PARALLEL_ALGORITHMS@ else None
-config.libcxx_gdb               = "@LIBCXX_GDB@"
 
 # Code signing
 config.llvm_codesign_identity   = "@LLVM_CODESIGNING_IDENTITY@"
 
 site.addsitedir(os.path.join(config.libcxx_src_root, 'utils'))
 
 # name: The name of this test suite.
 config.name = 'libc++'

libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py

 #===----------------------------------------------------------------------===##
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===##
 """Commands used to automate testing gdb pretty printers.
 
 This script is part of a larger framework to test gdb pretty printers. It
 runs the program, detects test cases, checks them, and prints results.
 
 See gdb_pretty_printer_test.sh.cpp on how to write a test case.
 
 """
 
 from __future__ import print_function
 import re
 import gdb
 import sys
 
 test_failures = 0
 
 
 class CheckResult(gdb.Command):
 
     def __init__(self):
         super(CheckResult, self).__init__(
             "print_and_compare", gdb.COMMAND_DATA)
 
     def invoke(self, arg, from_tty):
         try:
             # Stack frame is:
             # 0. StopForDebugger
             # 1. ComparePrettyPrintToChars or ComparePrettyPrintToRegex
             # 2. TestCase
             compare_frame = gdb.newest_frame().older()
             testcase_frame = compare_frame.older()
             test_loc = testcase_frame.find_sal()
             # Use interactive commands in the correct context to get the pretty
             # printed version
 
             value_str = self._get_value_string(compare_frame, testcase_frame)
 
             # Ignore the convenience variable name and newline
             value = value_str[value_str.find("= ") + 2:-1]
             gdb.newest_frame().select()
 
             expectation_val = compare_frame.read_var("expectation")
             check_literal = expectation_val.string(encoding="utf-8")
             if "PrettyPrintToRegex" in compare_frame.name():
                 test_fails = not re.match(check_literal, value)
             else:
                 test_fails = value != check_literal
 
             if test_fails:
                 global test_failures
                 print("FAIL: " + test_loc.symtab.filename +
                       ":" + str(test_loc.line))
                 print("GDB printed:")
                 print("   " + repr(value))
                 print("Value should match:")
                 print("   " + repr(check_literal))
                 test_failures += 1
             else:
                 print("PASS: " + test_loc.symtab.filename +
                       ":" + str(test_loc.line))
 
         except RuntimeError as e:
             # At this point, lots of different things could be wrong, so don't try to
             # recover or figure it out. Don't exit either, because then it's
             # impossible debug the framework itself.
             print("FAIL: Something is wrong in the test framework.")
             print(str(e))
             test_failures += 1
 
     def _get_value_string(self, compare_frame, testcase_frame):
         compare_frame.select()
         if "ComparePrettyPrint" in compare_frame.name():
             s = gdb.execute("p value", to_string=True)
         else:
             value_str = str(compare_frame.read_var("value"))
             clean_expression_str = value_str.strip("'\"")
             testcase_frame.select()
             s = gdb.execute("p " + clean_expression_str, to_string=True)
         if sys.version_info.major == 2:
             return s.decode("utf-8")
         return s
 
 
 def exit_handler(event=None):
     global test_failures
     if test_failures:
         print("FAILED %d cases" % test_failures)
     exit(test_failures)
 
 
 # Start code executed at load time
 
 # Disable terminal paging
 gdb.execute("set height 0")
 gdb.execute("set python print-stack full")
 test_failures = 0
 CheckResult()
 test_bp = gdb.Breakpoint("StopForDebugger")
 test_bp.enabled = True
 test_bp.silent = True
 test_bp.commands = "print_and_compare\ncontinue"
 # "run" won't return if the program exits; ensure the script regains control.
 gdb.events.exited.connect(exit_handler)
 gdb.execute("run")
 # If the program didn't exit, something went wrong, but we don't
 # know what. Fail on exit.
 test_failures += 1
 exit_handler(None)

libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp

 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// REQUIRES: libcxx_gdb
-//
+
+// REQUIRES: host-has-gdb
+// UNSUPPORTED: libcpp-has-no-localization
+
 // RUN: %{cxx} %{flags} %s -o %t.exe %{compile_flags} -g %{link_flags}
 // Ensure locale-independence for unicode tests.
-// RUN: %{libcxx_gdb} -nx -batch -iex "set autoload off" -ex "source %S/../../utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %S/gdb_pretty_printer_test.py" %t.exe
+// RUN: %{gdb} -nx -batch -iex "set autoload off" -ex "source %S/../../../utils/gdb/libcxx/printers.py" -ex "python register_libcxx_printer_loader()" -ex "source %S/gdb_pretty_printer_test.py" %t.exe
 
 #include <bitset>
 #include <deque>
 #include <list>
 #include <map>
 #include <memory>
 #include <queue>
 #include <set>

 // ComparePrettyPrintTo*.  Also, make sure neither it, nor the
 // variables we need present in the Compare functions are optimized
 // away.
 #ifdef TEST_COMPILER_GCC
 #define OPT_NONE __attribute__((noinline))
 #else
 #define OPT_NONE __attribute__((optnone))
 #endif
-void StopForDebugger(void *value, void *check) OPT_NONE;
-void StopForDebugger(void *value, void *check)  {}
+void StopForDebugger(void *, void *) OPT_NONE;
+void StopForDebugger(void *, void *)  {}
 
 
 // Prevents the compiler optimizing away the parameter in the caller function.
 template <typename Type>
-void MarkAsLive(Type &&t) OPT_NONE;
+void MarkAsLive(Type &&) OPT_NONE;
 template <typename Type>
-void MarkAsLive(Type &&t) {}
+void MarkAsLive(Type &&) {}
 
 // In all of the Compare(Expression)PrettyPrintTo(Regex/Chars) functions below,
 // the python script sets a breakpoint just before the call to StopForDebugger,
 // compares the result to the expectation.
 //
 // The expectation is a literal string to be matched exactly in
 // *PrettyPrintToChars functions, and is a python regular expression in
 // *PrettyPrintToRegex functions.
ComparePrettyPrintToChars(
       test1, "std::fpos with stream offset:0 with state: {count:0 value:0}");
   std::unique_ptr<char[]> buffer(new char[5]);
   input.read(buffer.get(), 5);
   test1 = input.tellg();
   ComparePrettyPrintToChars(
       test1, "std::fpos with stream offset:5 with state: {count:0 value:0}");
 }
 
-int main(int argc, char* argv[]) {
+int main(int, char**) {
   framework_self_test();
 
   string_test();
   a_namespace::string_view_test();
 
   u32string_test();
   tuple_test();
   unique_ptr_test();

libcxx/utils/libcxx/test/config.py

def configure_features(self):
             if self.cxx_stdlib_under_test == 'libc++':
                 # LIBCXX-WINDOWS-FIXME is the feature name used to XFAIL the
                 # initial Windows failures until they can be properly diagnosed
                 # and fixed. This allows easier detection of new test failures
                 # and regressions. Note: New failures should not be suppressed
                 # using this feature. (Also see llvm.org/PR32730)
                 self.config.available_features.add('LIBCXX-WINDOWS-FIXME')
 
-        libcxx_gdb = self.get_lit_conf('libcxx_gdb')
-        if libcxx_gdb and 'NOTFOUND' not in libcxx_gdb:
-            self.config.available_features.add('libcxx_gdb')
-            self.cxx.libcxx_gdb = libcxx_gdb
-
         target_triple = getattr(self.config, 'target_triple', None)
         if target_triple:
             if re.match(r'^x86_64.*-apple', target_triple):
                 self.config.available_features.add('x86_64-apple')
             if re.match(r'^x86_64.*-linux', target_triple):
                 self.config.available_features.add('x86_64-linux')
             if re.match(r'^i.86.*', target_triple):
                 self.config.available_features.add('target-x86')
def configure_substitutions(self):
         codesign_ident = self.get_lit_conf('llvm_codesign_identity', '')
         env_vars = ' '.join('%s=%s' % (k, pipes.quote(v)) for (k, v) in self.exec_env.items())
         exec_args = [
             '--execdir %T',
             '--codesign_identity "{}"'.format(codesign_ident),
             '--env {}'.format(env_vars)
         ]
         sub.append(('%{exec}', '{} {} -- '.format(self.executor, ' '.join(exec_args))))
-        if self.get_lit_conf('libcxx_gdb'):
-            sub.append(('%{libcxx_gdb}', self.get_lit_conf('libcxx_gdb')))
 
     def configure_triple(self):
         # Get or infer the target triple.
         target_triple = self.get_lit_conf('target_triple')
 
         # If no target triple was given, try to infer it from the compiler
         # under test.
         if not target_triple:

libcxx/utils/libcxx/test/dsl.py

def applyTo(self, config):
     # Use -Werror to make sure we see an error about the flag being unsupported.
     if hasCompileFlag(config, '-Werror ' + flag):
       config.substitutions = _addToSubstitution(config.substitutions, '%{compile_flags}', flag)
 
   def pretty(self, config, litParams):
     return 'add {} to %{{compile_flags}}'.format(self._getFlag(config))
 
 
+class AddSubstitution(ConfigAction):
+  """
+  This action adds the given substitution to the Lit configuration.
+
+  The substitution can be a string or a callable, in which case it is called
+  with the configuration to produce the actual substitution (as a string).
+  """
+  def __init__(self, key, substitution):
+    self._key = key
+    self._getSub = lambda config: substitution(config) if callable(substitution) else substitution
+
+  def applyTo(self, config):
+    key = self._key
+    sub = self._getSub(config)
+    config.substitutions.append((key, sub))
+
+  def pretty(self, config, litParams):
+    return 'add substitution {} = {}'.format(self._key, self._getSub(config))
+
+
 class Feature(object):
   """
   Represents a Lit available feature that is enabled whenever it is supported.
 
   A feature like this informs the test suite about a capability of the compiler,
   platform, etc. Unlike Parameters, it does not make sense to explicitly
   control whether a Feature is enabled -- it should be enabled whenever it
   is supported.

libcxx/utils/libcxx/test/features.py

 #===----------------------------------------------------------------------===##
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===##
 
 from libcxx.test.dsl import *
 import re
+import shutil
 import sys
 
 _isClang      = lambda cfg: '__clang__' in compilerMacros(cfg) and '__apple_build_version__' not in compilerMacros(cfg)
 _isAppleClang = lambda cfg: '__apple_build_version__' in compilerMacros(cfg)
 _isGCC        = lambda cfg: '__GNUC__' in compilerMacros(cfg) and '__clang__' not in compilerMacros(cfg)
 
 DEFAULT_FEATURES = [
   Feature(name='fcoroutines-ts',

 DEFAULT_FEATURES += [
   Feature(name='darwin', when=lambda cfg: '__APPLE__' in compilerMacros(cfg)),
   Feature(name='windows', when=lambda cfg: '_WIN32' in compilerMacros(cfg)),
   Feature(name='linux', when=lambda cfg: '__linux__' in compilerMacros(cfg)),
   Feature(name='netbsd', when=lambda cfg: '__NetBSD__' in compilerMacros(cfg))
 ]
 
 
+# Detect whether GDB is on the system, and if so add a substitution to access it.
+DEFAULT_FEATURES += [
+  Feature(name='host-has-gdb',
+    when=lambda cfg: shutil.which('gdb') is not None,
+    actions=[AddSubstitution('%{gdb}', lambda cfg: shutil.which('gdb'))]
+  )
+]
+
+
 # When vendor-specific availability annotations are enabled, add Lit features
 # with various forms of the target triple to make it easier to write XFAIL or
 # UNSUPPORTED markup for tests that are known to fail on a particular triple.
 #
 # More specifically, when the `use_system_cxx_lib` parameter is enabled, then
 # assuming the `target_triple` is set to `x86_64-apple-macosx10.12`, the
 # following features will be made available:
 #   - with_system_cxx_lib=macosx