[test-suite] Signal error if llvm-lit was not found

[lebedev.ri]

Summary

I'm not sure if that was intentional or not?
I found it quite surprising when trying to integrate a new external project.

[lebedev.ri]

Ping?

[hfinkel]

LGTM

[uweigand]

This seems to have broken test-suite build bots?

http://lab.llvm.org:8011/builders/clang-s390x-linux-lnt/builds/13870

CMake Error at CMakeLists.txt:220 (message):
  llvm-lit not found.

  Make sure it is in your path or specify it via -DTEST_SUITE_LIT:FILEPATH=""

The test suite seems to have executed correctly before this change brought in this error message ...

[lebedev.ri]

In D60091#1559406, @uweigand wrote:

This seems to have broken test-suite build bots?

http://lab.llvm.org:8011/builders/clang-s390x-linux-lnt/builds/13870

CMake Error at CMakeLists.txt:220 (message):
  llvm-lit not found.

  Make sure it is in your path or specify it via -DTEST_SUITE_LIT:FILEPATH=""

The test suite seems to have executed correctly before this change brought in this error message ...

Uhm, that makes no sense.
How is that test runs, not via check target?

test-suite/trunk/CMakeLists.txt

317–318

that is created here?

[lebedev.ri]

As discussed in IRC, the test-suite builds invoke lit manually; so it was always "broken".
I'll revert this for a bit, but please fix the configs.
Encountering it when trying to run check target after successful cmake configure was not pleasant at all.

[uweigand]

In D60091#1559418, @lebedev.ri wrote:

Uhm, that makes no sense.
How is that test runs, not via check target?

The builder is created by this rule:

'factory' : ClangBuilder.getClangCMakeBuildFactory(jobs=4,
                                                   clean=False,
                                                   checkout_lld=False,
                                                   useTwoStage=False,
                                                   runTestSuite=True,
                                                   stage1_config='Release',
                                                   testsuite_flags=['--threads=4', '--build-threads=4'],
                                                   extra_cmake_args=["-DLLVM_ENABLE_ASSERTIONS=ON"]),

That code seems to pass in the pass to llvm-lit via the "--use-lit" argument to "lnt":

lit = WithProperties('%(workdir)s/'+stage1_build+'/bin/llvm-lit')
test_suite_cmd = [python, lnt, 'runtest', 'test-suite',
                  '--no-timestamp',
                  '--sandbox', sandbox,
                  '--test-suite', test_suite_dir,
                  '--cc', cc,
                  '--cxx', cxx,
                  '--use-lit', lit]

This code is from zorg/buildbot/builders/ClangBuilder.py in http://llvm.org/svn/llvm-project/zorg/trunk ... is this not how it's supposed to be used?

[lebedev.ri]

Reverted in rL364448, thanks.
I have never touched bot configs, so i'm not the one to fix the phenomenon, so please address it.

[hfinkel]

In D60091#1559434, @lebedev.ri wrote:

Reverted in rL364448, thanks.
I have never touched bot configs, so i'm not the one to fix the phenomenon, so please address it.

I think that the point is that you can configure and build the test suite without lit, and if lnt knows there lit is, then cmake doesn't need to know. Is that right?

[lebedev.ri]

Oh i see, LNT itself does not propagate it's --use-lit to test-suite's TEST_SUITE_LIT.

test-suite/trunk/CMakeLists.txt

 
 option(TEST_SUITE_COLLECT_CODE_SIZE "Measure code size of binaries" ON)
 if(TEST_SUITE_COLLECT_CODE_SIZE)
   find_program(TEST_SUITE_LLVM_SIZE NAMES "llvm-size"
                HINTS ${CMAKE_C_COMPILER_DIRECTORY})
   mark_as_advanced(TEST_SUITE_LLVM_SIZE)
   if(TEST_SUITE_LLVM_SIZE STREQUAL "TEST_SUITE_LLVM_SIZE-NOTFOUND")
     message(FATAL_ERROR "llvm-size not found.
-Make sure it is in your path or set TEST_SUITE_COLLECT_CODE_SIZE to OFF")
+Make sure it is in your path or specify it via -DTEST_SUITE_LLVM_SIZE:FILEPATH=\"\" or set TEST_SUITE_COLLECT_CODE_SIZE to OFF")
   endif()
 endif()
 
 option(TEST_SUITE_USE_IR_PGO
   "Use IR PGO instrumentation (requires TEST_SUITE_PROFILE_GENERATE)" OFF)
 
 # Enable profile generate mode in lit. Note that this does not automatically
 # add something like -fprofile-instr-generate to the compiler flags.
 option(TEST_SUITE_PROFILE_GENERATE "Enable lit profile generate mode" OFF)
 # Set value to python style True/False
 if(TEST_SUITE_PROFILE_GENERATE)
   find_program(TEST_SUITE_LLVM_PROFDATA NAMES "llvm-profdata"
                HINTS ${CMAKE_C_COMPILER_DIRECTORY})
   mark_as_advanced(TEST_SUITE_LLVM_PROFDATA)
   if(TEST_SUITE_LLVM_PROFDATA STREQUAL "TEST_SUITE_LLVM_PROFDATA-NOTFOUND")
     message(FATAL_ERROR "llvm-profdata not found.
-Make sure it is in your path or set TEST_SUITE_PROFILE_GENERATE to OFF")
+Make sure it is in your path or specify it via -DTEST_SUITE_LLVM_PROFDATA:FILEPATH=\"\" or set TEST_SUITE_PROFILE_GENERATE to OFF")
   endif()
 
   set(TEST_SUITE_PROFILE_GENERATE "True")
 
   set(profile_instrumentation_flags -fprofile-instr-generate)
   if(TEST_SUITE_USE_IR_PGO)
     set(profile_instrumentation_flags -fprofile-generate)
   endif()

 if(ARCH STREQUAL "Mips")
   check_symbol_exists(__mips16 "" MIPS_IS_MIPS16_ENABLED)
   check_symbol_exists(__mips64 "" MIPS_IS_MIPS64_ENABLED)
 endif()
 
 find_program(TEST_SUITE_LIT NAMES "lit" "llvm-lit")
 set(TEST_SUITE_LIT_FLAGS "-sv" CACHE STRING "Flags used when running lit")
 mark_as_advanced(TEST_SUITE_LIT TEST_SUITE_LIT_FLAGS)
-mark_as_advanced(TEST_SUITE_LIT)
+if(TEST_SUITE_LIT STREQUAL "TEST_SUITE_LIT-NOTFOUND")
+  message(FATAL_ERROR "llvm-lit not found.
+Make sure it is in your path or specify it via -DTEST_SUITE_LIT:FILEPATH=\"\"")
+endif()
 
 add_subdirectory(tools)
 # Shortcut for the path to the fpcmp executable
 set(FPCMP ${CMAKE_BINARY_DIR}/tools/fpcmp)
 
 option(TEST_SUITE_COLLECT_COMPILE_TIME
        "Measure compile time by wrapping compiler invocations in timeit" ON)
 if(TEST_SUITE_COLLECT_COMPILE_TIME)

 if(TEST_SUITE_COLLECT_STATS)
   list(APPEND LIT_MODULES stats)
 endif()
 
 # Produce lit.site.cfg
 configure_file("${PROJECT_SOURCE_DIR}/lit.site.cfg.in" "${CMAKE_BINARY_DIR}/lit.site.cfg")
 
 get_property(TEST_SUITE_TARGETS GLOBAL PROPERTY TEST_SUITE_TARGETS)
 add_custom_target(check
   COMMAND ${TEST_SUITE_LIT} ${TEST_SUITE_LIT_FLAGS} .
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${TEST_SUITE_TARGETS}
   USES_TERMINAL
   )