[SPEC2017] Copy reference files to run dir, use for compares.

[fhahn]

Summary

When running the benchmarks on a different device, we need to copy the
reference files to the device and use the correct path for comparing on
the device.

We copy the output files (and compare files, if exist) to
${RUN_${_run_type}_DIR}/compares, which will be copied over to the
device, if one is configured. This is similar to how input files are
handled.

The validation commands have been updated to use the reference files
from the 'compare' directory in the run directory.

The patch also drops special handling for 'WORKDIR input', which
got translated to the input directory in the source folder. By dropping
all uses of WORKDIR input, the inputs from the run directory are used,
which should work as expected when running the benchmarks locally and on
a remote device.

[fhahn]

Update CFP2017 benchmarks.

[Meinersbur]

External/SPEC/CINT2017rate/557.xz_r/CMakeLists.txt

25

Why does this need a custom definition of RUN_INPUT_all_DIR ?

101

Why does this need a custom RUN_INPUT_train_DIR?

External/SPEC/SpecCPU2017.cmake

217

Did you consider removing the WORKDIR argument entirely? Seems that we never want to use a directory other than in the build dir. Are that other uses used of WORKDIR left?

[fhahn]

Thanks Michael! I removed the unused WORKDIR argument completely and also dropped the left-over RUN_INPUT__DIRs.

[fhahn]

External/SPEC/CINT2017rate/557.xz_r/CMakeLists.txt

25

Thanks, this was left over from an earlier iteration, I removed it.

101

Thanks, this was left over from an earlier iteration, I removed it.

External/SPEC/SpecCPU2017.cmake

217

That's a great point! It's not needed by any benchmark. I removed it.

[Meinersbur]

LGTM, thanks.

External/SPEC/CFP2017rate/508.namd_r/CMakeLists.txt

 # https://www.spec.org/cpu2017/Docs/benchmarks/508.namd_r.html
 include(../../SpecCPU2017.cmake)
 if (NOT TEST_SUITE_SPEC2017_ROOT)
   return ()
 endif ()
 
 speccpu2017_benchmark(RATE)
 
 add_definitions(-DNAMD_DISABLE_SSE)
 
 ## test ########################################################################
 
 speccpu2017_run_test(
-  --input "${INPUT_all_DIR}/apoa1.input"
+  --input "${RUN_test_DIR}/apoa1.input"
   --iterations 1 --output apoa1.test.output
   RUN_TYPE test
 )
 
 ## train #######################################################################
 
 speccpu2017_run_test(
-  --input "${INPUT_all_DIR}/apoa1.input"
+  --input "${RUN_train_DIR}/apoa1.input"
   --iterations 7 --output apoa1.train.output
   RUN_TYPE train
 )
 
 ## ref #########################################################################
 
 speccpu2017_run_test(
-  --input "${INPUT_all_DIR}/apoa1.input"
+  --input "${RUN_ref_DIR}/apoa1.input"
   --output apoa1.ref.output --iterations 65
   RUN_TYPE ref
 )
 
 ################################################################################
 
 speccpu2017_verify_output(ABSOLUTE_TOLERANCE 0.00005)
 speccpu2017_add_executable()
+speccpu2017_prepare_rundir()

External/SPEC/CFP2017rate/519.lbm_r/CMakeLists.txt

 # https://www.spec.org/cpu2017/Docs/benchmarks/519.lbm_r.html
 include(../../SpecCPU2017.cmake)
 if (NOT TEST_SUITE_SPEC2017_ROOT)
   return ()
 endif ()
 
 speccpu2017_benchmark(RATE)
 
 link_libraries(-lm)
 
 ## test ########################################################################
 
 speccpu2017_run_test(
   20 reference.dat 0 1 100_100_130_cf_a.of
   STDOUT lbm.out
-  WORKDIR input
   RUN_TYPE test
   SUITE_TYPE rate
 )
 
 ## train #######################################################################
 
 speccpu2017_run_test(
   300 reference.dat 0 1 100_100_130_cf_b.of
   STDOUT lbm.out
-  WORKDIR input
   RUN_TYPE train
   SUITE_TYPE rate
 )
 
 ## ref #########################################################################
 
 speccpu2017_run_test(
   3000 reference.dat 0 0 100_100_130_ldc.of
   STDOUT lbm.out
-  WORKDIR input
   RUN_TYPE ref
   SUITE_TYPE rate
 )
 
 ################################################################################
 
 # Using IGNORE_WHITESPACE because the reference output consistently uses tabs
 # while the program output also uses spaces for indention.
 speccpu2017_verify_output(ABSOLUTE_TOLERANCE 0.0000001 IGNORE_WHITESPACE)
 speccpu2017_add_executable()
+speccpu2017_prepare_rundir()

External/SPEC/CFP2017rate/544.nab_r/CMakeLists.txt

 # https://www.spec.org/cpu2017/Docs/benchmarks/544.nab_r.html
 include(../../SpecCPU2017.cmake)
 if (NOT TEST_SUITE_SPEC2017_ROOT)
   return ()
 endif ()
 
 speccpu2017_benchmark(RATE)
 
 add_definitions(-DNOPERFLIB -DNOREDUCE)
 speccpu2017_add_include_dirs(specrand regex-alpha)
 link_libraries(-lm)
 
 ## test ########################################################################
 
 speccpu2017_run_test(
   hkrdenq 1930344093 1000
-  WORKDIR input
   STDOUT hkrdenq.out
   RUN_TYPE test
 )
 
 ## train #######################################################################
 
 speccpu2017_run_test(
   aminos 391519156 1000
-  WORKDIR input
   STDOUT aminos.out
   RUN_TYPE train
 )
 
 speccpu2017_run_test(
   gcn4dna 1850041461 300
-  WORKDIR input
   STDOUT gcn4dna.out
   RUN_TYPE train
 )
 
 ## ref #########################################################################
 
 speccpu2017_run_test(
   1am0 1122214447 122
-  WORKDIR input
   STDOUT 1am0.out
   RUN_TYPE ref
   SUITE_TYPE rate
 )
 
 ################################################################################
 
 speccpu2017_verify_output(RELATIVE_TOLERANCE 0.01)
speccpu2017_add_executable(
   traceback.c
   chirvol.c
   specrand/specrand.c
   regex-alpha/regcomp.c
   regex-alpha/regerror.c
   regex-alpha/regexec.c
   regex-alpha/regfree.c
 )
+speccpu2017_prepare_rundir()

External/SPEC/CINT2017rate/523.xalancbmk_r/CMakeLists.txt

speccpu2017_add_include_dirs(
   xalanc/include
 )
 
 ## test ########################################################################
 
 speccpu2017_run_test(
   -v test.xml xalanc.xsl
   STDOUT test-test.out
-  WORKDIR input
   RUN_TYPE test
 )
 
 ## train #######################################################################
 
 speccpu2017_run_test(
   -v allbooks.xml xalanc.xsl
   STDOUT train-allbooks.out
-  WORKDIR input
   RUN_TYPE train
 )
 
 ## ref #########################################################################
 
 speccpu2017_run_test(
   -v t5.xml xalanc.xsl
   STDOUT ref-t5.out
-  WORKDIR input
   RUN_TYPE ref
 )
 
 ################################################################################
 
 speccpu2017_verify_output()
 speccpu2017_add_executable(
   AIXPlatformUtils.cpp
speccpu2017_add_executable(
   XercesWrapperHelper.cpp
   XercesWrapperNavigator.cpp
   XercesWrapperNavigatorAllocator.cpp
   XercesWrapperToXalanNodeMap.cpp
   XercesXPath.cpp
   YearDatatypeValidator.cpp
   YearMonthDatatypeValidator.cpp
 )
-
+speccpu2017_prepare_rundir()

External/SPEC/CINT2017rate/525.x264_r/CMakeLists.txt

set(ldecod_SourceNames
   ldecod_src/nalu.c
   ldecod_src/quant.c
   ldecod_src/intra8x8_pred.c
   ldecod_src/erc_api.c
   ldecod_src/header.c
   ldecod_src/biaridecod.c
   ldecod_src/input.c
 )
+speccpu2017_prepare_rundir()
 set(ldecod_Sources)
 foreach (_file IN LISTS ldecod_SourceNames)
   list(APPEND ldecod_Sources "${SRC_DIR}/${_file}")
 endforeach ()
 llvm_add_host_executable(ldecod_${SUFFIX}-host
   ldecod_${SUFFIX} ${ldecod_Sources}
   CPPFLAGS -I "${SRC_DIR}/ldecod_src/inc" -DSPEC
   LDFLAGS -lm

External/SPEC/CINT2017rate/557.xz_r/CMakeLists.txt

speccpu2017_add_include_dirs(
   liblzma/common
   liblzma/check
   liblzma/simple
   liblzma/delta
   liblzma/lz
   liblzma/rangecoder
 )
 
 ## test ########################################################################
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 4 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 1548636 1555348 0
   STDOUT cpu2006docs.tar-4-0.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 4 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 1462248 -1 1
   STDOUT cpu2006docs.tar-4-1.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 4 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 1428548 -1 2
   STDOUT cpu2006docs.tar-4-2.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
    cpu2006docs.tar.xz 4 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 1034828 -1 3e
    STDOUT cpu2006docs.tar-4-3e.out
-   WORKDIR "${INPUT_all_DIR}"
    RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 4 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 1061968 -1 4
   STDOUT cpu2006docs.tar-4-4.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 4 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 1034588 -1 4e
   STDOUT cpu2006docs.tar-4-4e.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 1 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 650156 -1 0
   STDOUT cpu2006docs.tar-1-0.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 1 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 639996 -1 1
   STDOUT cpu2006docs.tar-1-1.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 1 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 637616 -1 2
   STDOUT cpu2006docs.tar-1-2.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 1 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 628996 -1 3e
   STDOUT cpu2006docs.tar-1-3e.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 1 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 631912 -1 4
   STDOUT cpu2006docs.tar-1-4.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 1 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 629064 -1 4e
   STDOUT cpu2006docs.tar-1-4e.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE test
 )
 
 ## train #######################################################################
 
 speccpu2017_run_test(
   input.combined.xz 40 a841f68f38572a49d86226b7ff5baeb31bd19dc637a922a972b2e6d1257a890f6a544ecab967c313e370478c74f760eb229d4eef8a8d2836d233d3e9dd1430bf 6356684 -1 8
   STDOUT input.combined-40-8.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE train
 )
 
 speccpu2017_run_test(
   IMG_2560.cr2.xz 40 ec03e53b02deae89b6650f1de4bed76a012366fb3d4bdc791e8633d1a5964e03004523752ab008eff0d9e693689c53056533a05fc4b277f0086544c6c3cbbbf6 40822692 40824404 4
   STDOUT IMG_2560.cr2-40-4.out
-  WORKDIR input
   RUN_TYPE train
 )
 
 ## ref #########################################################################
 
 speccpu2017_run_test(
   cld.tar.xz 160 19cf30ae51eddcbefda78dd06014b4b96281456e078ca7c13e1c0c9e6aaea8dff3efb4ad6b0456697718cede6bd5454852652806a657bb56e07d61128434b474 59796407 61004416 6
   STDOUT cld.tar-160-6.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE ref
   SUITE_TYPE rate
 )
 
 speccpu2017_run_test(
   cpu2006docs.tar.xz 250 055ce243071129412e9dd0b3b69a21654033a9b723d874b2015c774fac1553d9713be561ca86f74e4f16f22e664fc17a79f30caa5ad2c04fbc447549c2810fae 23047774 23513385 6e
   STDOUT cpu2006docs.tar-250-6e.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE ref
   SUITE_TYPE rate
 )
 
 speccpu2017_run_test(
   input.combined.xz 250 a841f68f38572a49d86226b7ff5baeb31bd19dc637a922a972b2e6d1257a890f6a544ecab967c313e370478c74f760eb229d4eef8a8d2836d233d3e9dd1430bf 40401484 41217675 7
   STDOUT input.combined-250-7.out
-  WORKDIR "${INPUT_all_DIR}"
   RUN_TYPE ref
   SUITE_TYPE rate
 )
 
 ################################################################################
 
 speccpu2017_verify_output()
 speccpu2017_add_executable(
speccpu2017_add_executable(
     common/tuklib_progname.c
     common/tuklib_exit.c
     common/tuklib_cpucores.c
     common/tuklib_mbstr_width.c
     common/tuklib_mbstr_fw.c
     spec_mem_io/spec_mem_io.c
     sha-2/sha512.c
 )
+speccpu2017_prepare_rundir()

External/SPEC/CINT2017rate/999.specrand_ir/CMakeLists.txt

speccpu2017_run_test(
   STDOUT rand.234923.out
   RUN_TYPE ref
 )
 
 ################################################################################
 
 speccpu2017_verify_output()
 speccpu2017_add_executable()
+speccpu2017_prepare_rundir()

External/SPEC/SpecCPU2017.cmake

 # Add a "RUN:" line.
 #
 # RUN_TYPE   (test,train or ref)
 #            Only run if this TEST_SUITE_RUN_TYPE is is selected.
 #
 # SUITE_TYPE (rate or speed)
 #            Only run in the _r or _s benchmark suites.
 #
-# WORKDIR    Working dir for the executable to run in.
-#            "input" means the dataset source directory. Does not require
-#                    copying the input data to the rundir, but the benchmark
-#                    must not write data there.
-#            If not defined, the run_{run_type} directory is chosen.
-#
 # STDOUT     Write the benchmark's stdout into this file in the rundir.
 #
 # STDERR     Write the benchmark's stderr into this file in the rundir.
 #
 # ARGN       Benchmark's command line arguments
 macro (speccpu2017_run_test)
   cmake_parse_arguments(_arg
-    "" "RUN_TYPE;SUITE_TYPE;WORKDIR;STDOUT;STDERR" "" ${ARGN})
+    "" "RUN_TYPE;SUITE_TYPE;STDOUT;STDERR" "" ${ARGN})
 
   if ((NOT DEFINED _arg_SUITE_TYPE) OR
       (BENCHMARK_SUITE_TYPE IN_LIST _arg_SUITE_TYPE))
     if ((NOT DEFINED _arg_RUN_TYPE) OR
         (_arg_RUN_TYPE IN_LIST TEST_SUITE_RUN_TYPE))
 
       set(_stdout)
       if (DEFINED _arg_STDOUT)
         set(_stdout > "${RUN_${_arg_RUN_TYPE}_DIR}/${_arg_STDOUT}")
       endif ()
 
       set(_stderr)
       if (DEFINED _arg_STDERR)
         set(_stderr 2> "${RUN_${_arg_RUN_TYPE}_DIR}/${_arg_STDERR}")
       endif ()
 
-      set(_executable)
-      if (NOT DEFINED _arg_WORKDIR)
-        set(_workdir "${RUN_${_arg_RUN_TYPE}_DIR}")
-
-        # perlbench, xalancbmk need to be invoked with relative paths
-        # (SPEC made modifications that prepend another path to find the rundir)
-        file(RELATIVE_PATH _executable
           "${_workdir}" "${CMAKE_CURRENT_BINARY_DIR}/${PROG}")
-        set (_executable EXECUTABLE "${_executable}")
-      elseif (_arg_WORKDIR STREQUAL "input")
-        set(_workdir "${INPUT_${_arg_RUN_TYPE}_DIR}")
-      else ()
-        set(_workdir "${_arg_WORKDIR}")
-      endif ()
 
       llvm_test_run(
         ${_arg_UNPARSED_ARGUMENTS} ${_stdout} ${_stderr}
         RUN_TYPE ${_arg_RUN_TYPE}
         WORKDIR "${_workdir}"
         ${_executable}
       )
     endif ()
macro(speccpu2017_validate_image _imgfile _cmpfile _outfile)
   endif ()
 
   if ((NOT DEFINED _carg_SUITE_TYPE) OR (${BENCHMARK_SUITE_TYPE} IN_LIST _carg_SUITE_TYPE))
     get_filename_component(_basename "${_imgfile}" NAME_WE)
     get_filename_component(_ext "${_imgfile}" EXT)
     llvm_test_verify(
       cd "${RUN_${_carg_RUN_TYPE}_DIR}" &&
       "${CMAKE_CURRENT_BINARY_DIR}/${VALIDATOR}" ${_carg_UNPARSED_ARGUMENTS}
-        "${_imgfile}" "${DATA_${_carg_RUN_TYPE}_DIR}/compare/${_cmpfile}"
+        "${_imgfile}" "${RUN_${_carg_RUN_TYPE}_DIR}/compare/${_cmpfile}"
         > ${RUN_${_carg_RUN_TYPE}_DIR}/${_outfile}
       RUN_TYPE ${_carg_RUN_TYPE}
     )
   endif ()
 endmacro ()
 
 # Add a "VERIFY:" line that compares all of the benchmark's reference outputs
 # with files in the rundir.
macro(speccpu2017_verify_output)
   set(_ignorewhitespace)
   if (DEFINED _arg_IGNORE_WHITESPACE)
     set(_ignorewhitespace "-i")
   endif ()
 
   foreach (_runtype IN LISTS TEST_SUITE_RUN_TYPE ITEMS all)
     file(GLOB_RECURSE _reffiles "${OUTPUT_${_runtype}_DIR}/*")
     foreach (_reffile IN LISTS _reffiles)
-      file(RELATIVE_PATH _relfile "${OUTPUT_${_runtype}_DIR}" "${_reffile}")
-      set(_outfile "${RUN_${_runtype}_DIR}/${_relfile}")
+      file(RELATIVE_PATH _filename "${OUTPUT_${_runtype}_DIR}" "${_reffile}")
+      set(_outfile "${RUN_${_runtype}_DIR}/${_filename}")
+      set(_comparefile "${RUN_${_runtype}_DIR}/compare/${_filename}")
       llvm_test_verify(RUN_TYPE ${_runtype}
         "${FPCMP}" ${_abstol} ${_reltol} ${_ignorewhitespace}
-          "${_reffile}" "${_outfile}"
+          "${_comparefile}" "${_outfile}"
       )
     endforeach ()
   endforeach ()
 endmacro()
 
 
 # Add a SPEC CPU 2017 benchmark.
 #
macro(speccpu2017_add_executable)
   llvm_test_executable(${PROG} ${_sources})
 
   if (TARGET ${VALIDATOR})
     add_dependencies(${PROG} ${VALIDATOR})
   endif ()
 endmacro()
 
 
-# Copy the input data to the rundir.
+# Copy the input and comparison data to the rundir.
 #
 # Can often be avoided by either passing an absolute path to the file in the
 # input dir, or using the input dir as working directory and specify the output
 # file as an absolute path to the rundir.
 macro(speccpu2017_prepare_rundir)
   foreach (_runtype IN LISTS TEST_SUITE_RUN_TYPE)
     if (EXISTS "${INPUT_all_DIR}")
       llvm_copy_dir(${PROG} "${RUN_${_runtype}_DIR}" "${INPUT_all_DIR}")
     endif ()
     llvm_copy_dir(${PROG} "${RUN_${_runtype}_DIR}" "${INPUT_${_runtype}_DIR}")
+
+    file(MAKE_DIRECTORY "${RUN_${_runtype}_DIR}/compare")
+    llvm_copy_dir(${PROG} "${RUN_${_runtype}_DIR}/compare" "${OUTPUT_${_runtype}_DIR}")
+    if (EXISTS "${OUTPUT_${_runtype}_DIR}/../compare")
+        llvm_copy_dir(${PROG} "${RUN_${_runtype}_DIR}/compare" "${OUTPUT_${_runtype}_DIR}/../compare")
+    endif ()
   endforeach ()
 endmacro()
 
 # Run specpp on a list of SRCS provided and add them to the TARGET.
 # Three arguments are required:
 # TARGET must be already be defined prior to calling this command.
 # SRCS is provided usually via a GLOB command external to this macro.
 # DEFS are the arguments that would normally be passed to specpp.