Suggest lldb-dotest to reproduce a failure.

[JDevlieghere]

Summary

Rather than trying to figure out what arguments to pass to dotest.py, suggest using lldb-dotest. Athough this obviously won't work for all invocations (i.e. when you're passing custom arguments to lldb-dotest) it will work in the most common case (which is unfortunately not true today).

[davide]

Can you commit the whitespace fixes separately?

[aprantl]

packages/Python/lldbsuite/test/dosep.py

123–126

What do you think about adding the full path to lldb-dotest? I usually have at least three concurrent checkouts of lldb at any time.

[JDevlieghere]

In D47062#1104558, @davide wrote:

Can you commit the whitespace fixes separately?

Sure.

packages/Python/lldbsuite/test/dosep.py

123–126

Me too, but how would we get that information here? On of lldb-dotest's goals is that you can execute it from anywhere, so you'd either have to pass it explicitly to dotest.py (either using a custom argument and cmake variable combo, or by assuming it lives in the same directory as the current lldb and appending -dotest) but that feels like it might not be worth it. Maybe there's another solution you had in mind?

[aprantl]

packages/Python/lldbsuite/test/dosep.py

123–126

The --executable option that dotest receives should point to an lldb that ought to have a matching lldb-dotest next to it. I don't think that that is particularly ugly.

[JDevlieghere]

• Use absolute path to lldb-dotest when --executable is passed.

[labath]

Looks fine to me, just make the path computation windows-compatible.

packages/Python/lldbsuite/test/dotest.py

386

On windows, this will end up being lldb.exe-dotest

[JDevlieghere]

Make this work for Windows

[JDevlieghere]

I'm going to hold off on this until we have decided what to do for D46005. If we run the test cases as separate lit invocations, then the test format is in a better position to report this information.

[labath]

In D47062#1109399, @JDevlieghere wrote:

I'm going to hold off on this until we have decided what to do for D46005. If we run the test cases as separate lit invocations, then the test format is in a better position to report this information.

It looks like that patch has gotten a bit stuck, and I'm not sure when/if I'll have the time to unstuck it. So, if you still want to go ahead with this approach, then feel free to commit it.

[JDevlieghere]

Abandoning this until we can replace the dotest driver with lit.

packages/Python/lldbsuite/test/dosep.py

 from six.moves import queue
 
 # Our packages and modules
 import lldbsuite
 import lldbsuite.support.seven as seven
 
 from . import configuration
 from . import dotest_args
+from . import lldbtest_config
 from lldbsuite.support import optional_with
 from lldbsuite.test_event import dotest_channels
 from lldbsuite.test_event.event_builder import EventBuilder
 from lldbsuite.test_event import formatter
 
 from .test_runner import process_control
 
 # Status codes for running command with timeout.
with output_lock:
         if not (RESULTS_FORMATTER and RESULTS_FORMATTER.is_using_terminal()):
             print(file=sys.stderr)
             print(output, file=sys.stderr)
             if timeout:
                 timeout_str = " (TIMEOUT)"
             else:
                 timeout_str = ""
             print("[%s FAILED]%s" % (name, timeout_str), file=sys.stderr)
-            print("Command invoked: %s" % ' '.join(command), file=sys.stderr)
+            print(
+               "Reproduce with: {} -f {}".format(lldbtest_config.dotestWrapper,
+                                                name),
+               file=sys.stderr)
         update_progress(name)
 
 
 def report_test_pass(name, output):
     global output_lock
     with output_lock:
         update_progress(name)
 

packages/Python/lldbsuite/test/dotest.py

def parseOptionsAndInitTestdirs():
 
     if args.framework:
         configuration.lldbFrameworkPath = args.framework
 
     if args.executable:
         # lldb executable is passed explicitly
         lldbtest_config.lldbExec = os.path.realpath(args.executable)
         if not is_exe(lldbtest_config.lldbExec):
             lldbtest_config.lldbExec = which(args.executable)
         if not is_exe(lldbtest_config.lldbExec):
             logging.error(
                     '%s is not a valid executable to test; aborting...',
                     args.executable)
             sys.exit(-1)
 
+    if args.dotest_wrapper:
+        lldbtest_config.dotestWrapper = os.path.realpath(args.dotest_wrapper)
+    else:
+        lldbtest_config.dotestWrapper = "lldb-dotest"
+
     if args.server:
         os.environ['LLDB_DEBUGSERVER_PATH'] = args.server
 
     if args.excluded:
         for excl_file in args.excluded:
             parseExclusion(excl_file)
 
     if args.p:
if ("GDB_REMOTE_LOG" in os.environ):
         ci.HandleCommand(
             "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " gdb-remote "
             + gdb_remote_log_option,
             res)
         if not res.Succeeded():
             raise Exception(
                 'log enable failed (check GDB_REMOTE_LOG env variable)')
 
-
-def getMyCommandLine():
-    return ' '.join(sys.argv)
-
 # ======================================== #
 #                                          #
 # Execution of the test driver starts here #
 #                                          #
 # ======================================== #
 
 
 def checkDsymForUUIDIsNotOn():
if not configuration.sdir_name:
         configuration.sdir_name = timestamp_started
     os.environ["LLDB_SESSION_DIRNAME"] = os.path.join(
         os.getcwd(), configuration.sdir_name)
 
     sys.stderr.write(
         "\nSession logs for test failures/errors/unexpected successes"
         " will go into directory '%s'\n" %
         configuration.sdir_name)
-    sys.stderr.write("Command invoked: %s\n" % getMyCommandLine())
+    sys.stderr.write("Command invoked: %s\n" % ' '.join(sys.argv))
 
     if not os.path.isdir(configuration.sdir_name):
         try:
             os.mkdir(configuration.sdir_name)
         except OSError as exception:
             if exception.errno != errno.EEXIST:
                 raise
 

packages/Python/lldbsuite/test/dotest_args.py

group.add_argument(
         '--framework',
         metavar='framework-path',
         help='The path to LLDB.framework')
     group.add_argument(
         '--executable',
         metavar='executable-path',
         help='The path to the lldb executable')
     group.add_argument(
+        '--dotest-wrapper',
+        dest='dotest_wrapper',
+        metavar='executable-path',
+        help='The path to the lldb-dotest executable')
+    group.add_argument(
         '--server',
         metavar='server-path',
         help='The path to the debug server executable to use')
     group.add_argument(
         '--out-of-tree-debugserver',
         dest='out_of_tree_debugserver',
         action='store_true',
         help='A flag to indicate an out-of-tree debug server is being used')

packages/Python/lldbsuite/test/lldbtest.py

def markUnexpectedSuccess(self, bugnumber):
             if bugnumber is None:
                 print("unexpected success", file=sbuf)
             else:
                 print(
                     "unexpected success (problem id:" + str(bugnumber) + ")",
                     file=sbuf)
 
     def getRerunArgs(self):
-        return " -f %s.%s" % (self.__class__.__name__, self._testMethodName)
+        return "-f %s.%s" % (self.__class__.__name__, self._testMethodName)
 
     def getLogBasenameForCurrentTest(self, prefix=None):
         """
         returns a partial path that can be used as the beginning of the name of multiple
         log files pertaining to this test
 
         <session-dir>/<arch>-<compiler>-<test-file>.<test-class>.<test-method>
         """
def dumpSessionInfo(self):
         # put footer (timestamp/rerun instructions) into session
         testMethod = getattr(self, self._testMethodName)
         if getattr(testMethod, "__benchmarks_test__", False):
             benchmarks = True
         else:
             benchmarks = False
 
         import datetime
-        print(
-            "Session info generated @",
+        print("Session info generated @",
             datetime.datetime.now().ctime(),
             file=self.session)
-        print(
-            "To rerun this test, issue the following command from the 'test' directory:\n",
+        print("To rerun this test, issue the following command:\n",
             file=self.session)
         print(
-            "./dotest.py %s -v %s %s" %
-            (self.getRunOptions(),
-             ('+b' if benchmarks else '-t'),
-                self.getRerunArgs()),
+            "{} {}".format(lldbtest_config.dotestWrapper, self.getRerunArgs()),
             file=self.session)
         self.session.close()
         del self.session
 
         # process the log files
         log_files_for_this_test = glob.glob(self.log_basename + "*")
 
         if prefix != 'Success' or lldbtest_config.log_success:
def expectedArch(self, archs):
             return True
 
         for arch in archs:
             if arch in self.getArchitecture():
                 return True
 
         return False
 
-    def getRunOptions(self):
-        """Command line option for -A and -C to run this test again, called from
-        self.dumpSessionInfo()."""
-        arch = self.getArchitecture()
-        comp = self.getCompiler()
-        option_str = ""
-        if arch:
-            option_str = "-A " + arch
-        if comp:
-            option_str += " -C " + comp
-        return option_str
-
     def getDebugInfo(self):
         method = getattr(self, self.testMethodName)
         return getattr(method, "debug_info", None)
 
     # ==================================================
     # Build methods supported through a plugin interface
     # ==================================================
 

packages/Python/lldbsuite/test/lldbtest_config.py

 # leave logs/traces even for successful test runs
 log_success = False
 
 # Indicate whether we're testing with an out-of-tree debugserver
 out_of_tree_debugserver = False
 
 # path to the lldb command line executable tool
 lldbExec = None
+
+# path to lldb-dotest wrapper
+dotestWrapper = None

utils/lldb-dotest/CMakeLists.txt

 # Make lldb-dotest a custom target.
 add_custom_target(lldb-dotest)
 add_dependencies(lldb-dotest ${LLDB_TEST_DEPS})
 
 get_property(LLDB_DOTEST_ARGS GLOBAL PROPERTY LLDB_DOTEST_ARGS_PROPERTY)
 
 # Generate wrapper for each build mode.
 if(NOT "${CMAKE_CFG_INTDIR}" STREQUAL ".")
   foreach(LLVM_BUILD_MODE ${CMAKE_CONFIGURATION_TYPES})
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_DOTEST_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_DOTEST_ARGS "${LLDB_DOTEST_ARGS}")
+    set(LLDB_DOTEST_WRAPPER ${LLDB_DOTEST_DIR}/lldb-dotest)
+    set(LLDB_DOTEST_ARGS_WITH_WRAPPER ${LLDB_DOTEST_ARGS})
+    list(APPEND LLDB_DOTEST_ARGS_WITH_WRAPPER --dotest-wrapper "${LLDB_DOTEST_WRAPPER}")
     configure_file(
       lldb-dotest.in
-      ${LLDB_DOTEST_DIR}/lldb-dotest
+      ${LLDB_DOTEST_WRAPPER}
       )
   endforeach()
 else()
+  set(LLDB_DOTEST_WRAPPER ${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb-dotest)
+  set(LLDB_DOTEST_ARGS_WITH_WRAPPER ${LLDB_DOTEST_ARGS})
+  list(APPEND LLDB_DOTEST_ARGS_WITH_WRAPPER --dotest-wrapper "${LLDB_DOTEST_WRAPPER}")
   configure_file(
     lldb-dotest.in
-    ${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb-dotest
+    ${LLDB_DOTEST_WRAPPER}
     )
 endif()

utils/lldb-dotest/lldb-dotest.in

 #!/usr/bin/env python
 import subprocess
 import sys
 
 dotest_path = '@LLDB_SOURCE_DIR@/test/dotest.py'
-dotest_args_str = '@LLDB_DOTEST_ARGS@'
+dotest_args_str = '@LLDB_DOTEST_ARGS_WITH_WRAPPER@'
 
 if __name__ == '__main__':
     wrapper_args = sys.argv[1:]
     dotest_args = dotest_args_str.split(';')
     # Build dotest.py command.
     cmd = [dotest_path, '-q']
     cmd.extend(dotest_args)
     cmd.extend(wrapper_args)
     # Invoke dotest.py and return exit code.
     sys.exit(subprocess.call(cmd))