[lldb] DWZ 15/17: New testsuite category 'dwz'

[jankratochvil]

My question about upstreaming of this patchset.

[jankratochvil]

I will be no longer involved with this patchset.

lldb/packages/Python/lldbsuite/test/builders/builder.py

def buildDsym(self,
                   architecture=None,
                   compiler=None,
                   dictionary=None,
                   testdir=None,
                   testname=None):
         # False signifies that we cannot handle building with dSYM.
         return False
 
+    def buildDWZ(self,
+                 sender=None,
+                 architecture=None,
+                 compiler=None,
+                 dictionary=None,
+                 testdir=None,
+                 testname=None):
+        """Build the binaries with DWZ - a DWARF optimization tool."""
+        commands = []
+        commands.append(
+            self.getMake(testdir, testname) + [
+                "MAKE_DSYM=NO", "DWZ=YES",
+                self.getArchCFlags(architecture),
+                self.getArchSpec(architecture),
+                self.getCCSpec(compiler),
+                self.getExtraMakeArgs(),
+                self.getSDKRootSpec(),
+                self.getModuleCacheSpec(),
+                self.getCmdLine(dictionary)
+            ])
+
+        self.runBuildCommands(commands, sender=sender)
+        # True signifies that we can handle building with DWZ.
+        return True
+
     def cleanup(self, sender=None, dictionary=None):
         """Perform a platform-specific cleanup after the test."""
         return True

lldb/packages/Python/lldbsuite/test/dotest.py

 def checkForkVForkSupport():
     from lldbsuite.test import lldbplatformutil
 
     platform = lldbplatformutil.getPlatform()
     if platform not in ["freebsd", "linux", "netbsd"]:
         configuration.skip_categories.append("fork")
 
 
+def canRunDWZTests():
+    from lldbsuite.test import lldbplatformutil
+
+    platform = lldbplatformutil.getPlatform()
+
+    if platform == "linux":
+        import distutils.spawn
+
+        if not os.access("/usr/lib/rpm/sepdebugcrcfix", os.X_OK):
+            return False, "Unable to find /usr/lib/rpm/sepdebugcrcfix"
+        if distutils.spawn.find_executable("eu-strip") is None:
+            return False, "Unable to find executable eu-strip"
+        if distutils.spawn.find_executable("dwz") is None:
+            return False, "Unable to find executable dwz"
+        return True, "/usr/lib/rpm/sepdebugcrcfix, eu-strip and dwz found"
+
+    return False, "Don't know how to build with DWZ on %s" % platform
+
+def checkDWZSupport():
+    result, reason = canRunDWZTests()
+    if result:
+        return # dwz supported
+    if "dwz" in configuration.categories_list:
+        return # dwz category explicitly requested, let it run.
+    print("dwz tests will not be run because: " + reason)
+    configuration.skip_categories.append("dwz")
+
 def run_suite():
     # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
     # does not exist before proceeding to running the test suite.
     if sys.platform.startswith("darwin"):
         checkDsymForUUIDIsNotOn()
 
     # Start the actions by first parsing the options while setting up the test
     # directories, followed by setting up the search paths for lldb utilities;
def run_suite():
 
     checkLibcxxSupport()
     checkLibstdcxxSupport()
     checkWatchpointSupport()
     checkDebugInfoSupport()
     checkDebugServerSupport()
     checkObjcSupport()
     checkForkVForkSupport()
+    checkDWZSupport()
 
     print("Skipping the following test categories: {}".format(configuration.skip_categories))
 
     for testdir in configuration.testdirs:
         for (dirpath, dirnames, filenames) in os.walk(testdir):
             visit('Test', dirpath, filenames)
 
     #

lldb/packages/Python/lldbsuite/test/lldbinline.py

def BuildMakefile(self):
 
             makefile.write("include Makefile.rules\n")
 
     def _test(self):
         self.BuildMakefile()
         self.build(dictionary=self._build_dict)
         self.do_test()
 
+    @add_test_categories(["dwz"])
+    def __test_with_dwz(self):
+        self.using_dsym = False
+        self.BuildMakefile()
+        self.build()
+        self.do_test()
+    __test_with_dwz.debug_info = "dwz"
+
     def execute_user_command(self, __command):
         exec(__command, globals(), locals())
 
     def _get_breakpoint_ids(self, thread):
         ids = set()
         for i in range(0, thread.GetStopReasonDataCount(), 2):
             ids.add(thread.GetStopReasonDataAtIndex(i))
         self.assertGreater(len(ids), 0)

lldb/packages/Python/lldbsuite/test/lldbtest.py

def buildGModules(
             raise Exception("NO_DEBUG_INFO_TESTCASE must build with buildDefault")
 
         module = builder_module()
         dictionary = lldbplatformutil.finalize_build_dictionary(dictionary)
         if not module.buildGModules(self, architecture, compiler,
                                     dictionary, testdir, testname):
             raise Exception("Don't know how to build binary with gmodules")
 
+    def buildDWZ(
+            self,
+            architecture=None,
+            compiler=None,
+            dictionary=None):
+        """Platform specific way to build binaries with dwz optimizer."""
+        testdir = self.mydir
+        testname = self.getBuildDirBasename()
+        if self.getDebugInfo() != "dwz":
+            raise Exception("NO_DEBUG_INFO_TESTCASE must build with buildDefault")
+
+        module = builder_module()
+        dictionary = lldbplatformutil.finalize_build_dictionary(dictionary)
+        if not module.buildDWZ(self, architecture, compiler,
+                               dictionary, testdir, testname):
+            raise Exception("Don't know how to build binary with dwz")
+
     def signBinary(self, binary_path):
         if sys.platform.startswith("darwin"):
             codesign_cmd = "codesign --force --sign \"%s\" %s" % (
                 lldbtest_config.codesign_identity, binary_path)
             call(codesign_cmd, shell=True)
 
     def findBuiltClang(self):
         """Tries to find and use Clang from the build directory as the compiler (instead of the system compiler)."""
def build(
         elif self.getDebugInfo() == "dsym":
             return self.buildDsym(architecture, compiler, dictionary)
         elif self.getDebugInfo() == "dwarf":
             return self.buildDwarf(architecture, compiler, dictionary)
         elif self.getDebugInfo() == "dwo":
             return self.buildDwo(architecture, compiler, dictionary)
         elif self.getDebugInfo() == "gmodules":
             return self.buildGModules(architecture, compiler, dictionary)
+        elif self.getDebugInfo() == "dwz":
+            return self.buildDWZ(architecture, compiler, dictionary)
         else:
             self.fail("Can't build for debug info: %s" % self.getDebugInfo())
 
     """Assert that an lldb.SBError is in the "success" state."""
     def assertSuccess(self, obj, msg=None):
         if not obj.Success():
             error = obj.GetCString()
             self.fail(self._formatMessage(msg,

lldb/packages/Python/lldbsuite/test/make/Makefile.rules

 #
 # Also might be of interest:
 # FRAMEWORK_INCLUDES (Darwin only) :=
 # CFLAGS_EXTRAS :=
 # LD_EXTRAS :=
 # SPLIT_DEBUG_SYMBOLS := YES
 # CROSS_COMPILE :=
 # USE_PRIVATE_MODULE_CACHE := YES
+# DWZ := YES
 
 # Uncomment line below for debugging shell commands
 # SHELL = /bin/sh -x
 
 # Suppress built-in suffix rules. We explicitly define rules for %.o.
 .SUFFIXES:
 
 SRCDIR := $(shell dirname $(firstword $(MAKEFILE_LIST)))
else
   ifeq "$(ARCH)" "s390x"
     override ARCH :=
     override ARCHFLAG :=
   endif
   ifeq "$(findstring mips,$(ARCH))" "mips"
     override ARCHFLAG := -
   endif
 
+  ifeq "$(DWZ)" "YES"
+    # DWZ always does SPLIT_DEBUG_SYMBOLS (as otherwise the dwz command fails)
+  else
     ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
       DSYM = $(EXE).debug
     endif
   endif
+endif
 
 LIMIT_DEBUG_INFO_FLAGS =
 NO_LIMIT_DEBUG_INFO_FLAGS =
 MODULE_DEBUG_INFO_FLAGS =
 ifneq (,$(findstring clang,$(CC)))
    LIMIT_DEBUG_INFO_FLAGS += -flimit-debug-info
    NO_LIMIT_DEBUG_INFO_FLAGS += -fno-limit-debug-info
    MODULE_DEBUG_INFO_FLAGS += -gmodules
ifneq "$(OS)" "Darwin"
   override AR = $(ARCHIVER)
 endif
 
 ifdef PIE
   LDFLAGS += -pie
 endif
 
 #----------------------------------------------------------------------
+# Make the DWZ symbol file from the executable if $(DWZ) = "YES"
+# The dwz retry without -m is there because:
+#   dwz -m still succeeds even for no debug info present (missing -g):
+#     dwz: $(1).debug: .debug_info section not present
+#   But dwz -m fails if there is some debug info but none if it is big enough
+#   to be shared from the the common file $(1).debug.dwz:
+#     dwz: $(1).debug.dwz: .debug_info section not present
+#----------------------------------------------------------------------
+ifeq "$(DWZ)" "YES"
+  dwz_strip = \
+    eu-strip --remove-comment -f "$(1).debug" "$(1)" \
+    && cp "$(1).debug" "$(1).debug.dup" \
+    && (dwz -m "$(1).debug.dwz" -M $$(basename "$(1).debug.dwz") "$(1).debug" "$(1).debug.dup" \
+        || dwz "$(1).debug") \
+    && /usr/lib/rpm/sepdebugcrcfix . "$$(realpath --relative-to=$$PWD "$(1)")"
+else
+  dwz_strip =
+endif
+
+#----------------------------------------------------------------------
 # Windows specific options
 #----------------------------------------------------------------------
 ifeq "$(OS)" "Windows_NT"
   ifneq (,$(findstring clang,$(CC)))
     # Clang for Windows doesn't support C++ Exceptions
     CXXFLAGS += -fno-exceptions
     CXXFLAGS += -D_HAS_EXCEPTIONS=0
 

 
 #----------------------------------------------------------------------
 # Compile the executable from all the objects.
 #----------------------------------------------------------------------
 ifneq "$(DYLIB_NAME)" ""
 ifeq "$(DYLIB_ONLY)" ""
 $(EXE) : $(OBJECTS) $(DYLIB_FILENAME)
   $(LD) $(OBJECTS) -L. -l$(DYLIB_NAME) $(LDFLAGS) -o "$(EXE)"
+  $(call dwz_strip,$(EXE))
 ifneq "$(CODESIGN)" ""
   $(CODESIGN) -s - "$(EXE)"
 endif
 else
 EXE = $(DYLIB_FILENAME)
 endif
 else
 $(EXE) : $(OBJECTS)
   $(LD) $(OBJECTS) $(LDFLAGS) -o "$(EXE)"
+  $(call dwz_strip,$(EXE))
 ifneq "$(CODESIGN)" ""
   $(CODESIGN) -s - "$(EXE)"
 endif
 endif
 
 #----------------------------------------------------------------------
 # Make the dSYM file from the executable if $(MAKE_DSYM) != "NO"
 #----------------------------------------------------------------------

 endif
 ifneq "$(MAKE_DSYM)" "NO"
 ifneq "$(DS)" ""
   "$(DS)" $(DSFLAGS) "$(DYLIB_FILENAME)"
 endif
 endif
 else
   $(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)"
+ifeq "$(DWZ)" "YES"
+  # DWZ always does SPLIT_DEBUG_SYMBOLS (as otherwise the dwz command fails)
+  $(call dwz_strip,$(DYLIB_FILENAME))
+else
 ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
   $(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug"
   $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)"
 endif
 endif
+endif
 
 #----------------------------------------------------------------------
 # Make the precompiled header and compile C++ sources against it
 #----------------------------------------------------------------------
 
 ifneq "$(PCH_OUTPUT)" ""
 $(PCH_OUTPUT) : $(PCH_CXX_SOURCE)
   $(CXX) $(CXXFLAGS) -x c++-header -o $@ $<

lldb/packages/Python/lldbsuite/test/test_categories.py

 
 # Third-party modules
 
 # LLDB modules
 from lldbsuite.support import gmodules
 
 
 debug_info_categories = [
-    'dwarf', 'dwo', 'dsym', 'gmodules'
+    'dwarf', 'dwo', 'dsym', 'gmodules', 'dwz'
 ]
 
 all_categories = {
     'basic_process': 'Basic process execution sniff tests.',
     'cmdline': 'Tests related to the LLDB command-line interface',
     'dataformatters': 'Tests related to the type command and the data formatters subsystem',
     'debugserver': 'Debugserver tests',
     'dsym': 'Tests that can be run with DSYM debug information',
     'dwarf': 'Tests that can be run with DWARF debug information',
     'dwo': 'Tests that can be run with DWO debug information',
+    'dwz': 'Tests using DWZ and its DWARF partial units',
     'dyntype': 'Tests related to dynamic type support',
     'expression': 'Tests related to the expression parser',
     'flakey': 'Flakey test cases, i.e. tests that do not reliably pass at each execution',
     'fork': 'Tests requiring the process plugin fork/vfork event support',
     'gmodules': 'Tests that can be run with -gmodules debug information',
     'instrumentation-runtime': 'Tests for the instrumentation runtime plugins',
     'libc++': 'Test for libc++ data formatters',
     'libstdcxx': 'Test for libstdcxx data formatters',
if category == "dwo":
         return platform in ["linux", "freebsd"]
     elif category == "dsym":
         return platform in ["darwin", "macosx", "ios", "watchos", "tvos", "bridgeos"]
     elif category == "gmodules":
         # First, check to see if the platform can even support gmodules.
         if platform not in ["freebsd", "darwin", "macosx", "ios", "watchos", "tvos", "bridgeos"]:
             return False
         return gmodules.is_compiler_clang_with_gmodules(compiler_path)
+    elif category == "dwz":
+        return platform in ["linux"]
     return True
 
 
 def validate(categories, exact_match):
     """
     For each category in categories, ensure that it's a valid category (if exact_match is false,
     unique prefixes are also accepted). If a category is invalid, print a message and quit.
        If all categories are valid, return the list of categories. Prefixes are expanded in the

lldb/test/API/lang/cpp/template-function/TestTemplateFunctions.py

def do_test_template_function(self, add_cast):
           self.expect_expr("d1 << d2", result_type="bool", result_value="true")
           self.expect_expr("d1 == d2", result_type="bool", result_value="true")
 
     @skipIfWindows
     def test_template_function_with_cast(self):
         self.do_test_template_function(True)
 
     @skipIfWindows
-    @expectedFailureAll(debug_info=["dwarf", "gmodules", "dwo"])
+    @expectedFailureAll(debug_info=["dwarf", "gmodules", "dwo", "dwz"])
     def test_template_function_without_cast(self):
         self.do_test_template_function(False)