RunSafely: Add option to omit exit status from output.

[MatzeB]

Summary

Add an option to stop RunSafely.sh from outputting the process exit status at the end of the output.

This is useful for external benchmark suites which bring reference files
for stdout without the llvm-test-suite convention of having an exit code
behind the stdout output.

We also make cmake/lit system configurable here: By default we do not append the exit status but we create a bunch of local lit configs to enable this behaviour for the SingleSource/MultiSource and some of the External tests in the test-suite.

test-suite/trunk/External/Nurbs/CMakeLists.txt

if(TEST_SUITE_NURBS_ROOT)
   llvm_test_run(/k all timed /t 500 /vsteps 192 /usteps 192 /vcp 20 /ucp 20)
   llvm_test_verify(${CMAKE_BINARY_DIR}/tools/fpcmp
     %o
     ${CMAKE_CURRENT_SOURCE_DIR}/nurbs.reference_output
   )
 
   llvm_multisource()
 endif()
+file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")

test-suite/trunk/External/Nurbs/lit.local.cfg

+config.output_append_exitstatus = True

test-suite/trunk/External/Povray/CMakeLists.txt

if(TEST_SUITE_POVRAY_ROOT)
   )
   llvm_test_verify(${CMAKE_BINARY_DIR}/tools/fpcmp
     %o
     ${CMAKE_CURRENT_SOURCE_DIR}/povray.reference_output
   )
 
   llvm_multisource()
 endif()
+file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")

test-suite/trunk/External/Povray/lit.local.cfg

+config.output_append_exitstatus = True

test-suite/trunk/External/skidmarks10/CMakeLists.txt

if(TEST_SUITE_SKIDMARKS10_ROOT)
 
   llvm_test_run(0 ${TEST_SUITE_SKIDMARKS10_ROOT}/golden)
   llvm_test_verify(${CMAKE_BINARY_DIR}/tools/fpcmp
     %o
     ${CMAKE_CURRENT_SOURCE_DIR}/skidmarks.reference_output
   )
   llvm_multisource()
 endif()
+file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")

test-suite/trunk/External/skidmarks10/lit.local.cfg

+config.output_append_exitstatus = True

test-suite/trunk/MultiSource/CMakeLists.txt

 # Autogenerated from MultiSource/Makefile
 
 set(DIRS)
 set(PARALLEL_DIRS)
 
 set(LEVEL ..)
 set(PARALLEL_DIRS Applications Benchmarks)
 if(NOT DEFINED BENCHMARKING_ONLY)
   list(APPEND PARALLEL_DIRS UnitTests)
 endif()
 llvm_multisource()
 
 llvm_add_subdirectories(${DIRS} ${PARALLEL_DIRS})
+
+file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")

test-suite/trunk/MultiSource/lit.local.cfg

+config.output_append_exitstatus = True

test-suite/trunk/RunSafely.sh

 #           contains a timing of the program and the program's exit code.
 #
 #           If optional parameters -r <remote host> -l <remote user> are
 #           specified, it execute the program remotely using rsh.
 #
 # Syntax:
 #
 #   RunSafely.sh [-r <rhost>] [-l <ruser>] [-rc <client>] [-rp <port>]
-#                [-u <under>] [--show-errors] -t <timeit>
+#                [-u <under>] [--show-errors] [--omit-exitval] -t <timeit>
 #                <timeout> <infile> <outfile> <program> <args...>
 #
 #   where:
 #     <rhost>   is the remote host to execute the program
 #     <ruser>   is the username on the remote host
 #     <client>  is the remote client used to execute the program
 #     <port>    is the port used by the remote client
 #     <under>   is a wrapper that the program is run under
 #     <timeit>  is a wrapper that is used to collect timing data
 #     <timeout> is the maximum number of seconds to let the <program> run
 #     <infile>  is a file from which standard input is directed
 #     <outfile> is a file to which standard output and error are directed
 #     <program> is the path to the program to run
 #     <args...> are the arguments to pass to the program.
 #
 # If --show-errors is given, then the output file will be printed if the command
 # fails (returns a non-zero exit code).
+# Unless --omit-exitval is given the last line of the outfile has the form
+# "exit NN" with NN being the exit status number of the program.
 
 if [ $# -lt 4 ]; then
   echo "./RunSafely.sh [-t <PATH>] <timeout> <infile> <outfile>" \
        "<program> <args...>"
   exit 1
 fi
 
 # Save a copy of the original arguments in a string before we
 # clobber them with the shift command.
 ORIG_ARGS="$*"
 
 DIR=${0%%`basename $0`}
 
 RHOST=
 RFLAGS=""
 RCLIENT=rsh
 RUN_UNDER=
 TIMEIT=
 SHOW_ERRORS=0
+OMIT_EXITVAL=0
+if [ $1 = "--omit-exitval" ]; then
+  OMIT_EXITVAL=1
+  shift 1
+fi
 if [ $1 = "-r" ]; then
   RHOST=$2
   shift 2
 fi
 if [ $1 = "-l" ]; then
   RFLAGS="$RFLAGS -l $2"
   shift 2
 fi
elif test "$exitval" -eq 128 ; then
   echo "TEST $PROGRAM FAILED: exit status 128!"
 elif test "$exitval" -gt 128 ; then
   echo "TEST $PROGRAM FAILED: process terminated by signal (exit status $exitval)!"
 elif [ "$SHOW_ERRORS" -eq 1 -a "$exitval" -ne 0 ] ; then
   echo "TEST $PROGRAM FAILED: EXIT != 0"
 else
   fail=no
 fi
+if [ "$OMIT_EXITVAL" -ne 1 ]; then
   echo "exit $exitval" >> $OUTFILE
+fi
 
 # If we detected a failure, print the name of the test executable to the
 # output file. This will cause it to compare as different with other runs
 # of the same test even if they fail in the same way, because they'll have
 # different command names.
 if [ "${fail}" != "no" ]; then
   echo "RunSafely.sh detected a failure with these command-line arguments: " \
        "$ORIG_ARGS" >> $OUTFILE

test-suite/trunk/SingleSource/CMakeLists.txt

 # Autogenerated from SingleSource/Makefile
 
 set(DIRS)
 set(PARALLEL_DIRS)
 
 set(LEVEL ..)
 set(PARALLEL_DIRS Regression Benchmarks)
 if(NOT DEFINED BENCHMARKING_ONLY)
   list(APPEND PARALLEL_DIRS UnitTests)
 endif()
 list(APPEND LDFLAGS -lm)
 llvm_singlesource()
 
 llvm_add_subdirectories(${DIRS} ${PARALLEL_DIRS})
+
+file(COPY lit.local.cfg DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")

test-suite/trunk/SingleSource/lit.local.cfg

+config.output_append_exitstatus = True

test-suite/trunk/lit.cfg

def prepareRunSafely(config, commandline, outfile):
 
     if stdin is None:
         stdin = "/dev/null"
     if stdout is not None or stderr is not None:
         raise Exception("stdout/stderr redirection in combination with RunSafely not implemented yet")
 
     runsafely = "%s/RunSafely.sh" % config.test_suite_root
     runsafely_prefix = [ runsafely ]
+    if not config.output_append_exitstatus:
+        runsafely_prefix += ["--omit-exitval"]
     if config.remote_host:
         runsafely_prefix += [ "-r", config.remote_host ]
         if config.remote_user:
             runsafely_prefix += [ "-l", config.remote_user ]
         if config.remote_client:
             runsafely_prefix += [ "-rc", config.remote_client ]
         if config.remote_port:
             runsafely_prefix += [ "-rp", config.remote_port ]
def execute(self, test, litConfig):
             pass
 
         return result
 
 config.name = 'test-suite'
 config.test_format = TestSuiteTest()
 config.suffixes = ['.test']
 config.excludes = ['ABI-Testsuite']
+config.output_append_exitstatus = False
 if 'SSH_AUTH_SOCK' in os.environ:
     config.environment['SSH_AUTH_SOCK'] = os.environ['SSH_AUTH_SOCK']