[analyzer] [NFC] Introduce separate targets for testing the analyzer: check-clang-analyzer and check-clang-analyzer-z3

[george.karpenkov]

Summary

Current testing setup for analyzer tests with Z3 is rather inconvenient:

• There's no way to run the analyzer tests separately (I use LIT_FILTER=Analysis ninja check-clang, but a direct target is nicer)
• When Clang is built with Z3 support, there's no way to *not* run tests with Z3 solver, and this is often desired, as tests with Z3 solver take a very long time.

This patch introduces two extra targets:

• check-clang-analyzer
• check-clang-analyzer-z3

which solve those problems.

[george.karpenkov]

Better variable names, better scoping.

[mikhail.ramalho]

I can't comment on the code but it works correctly for me.

I wish I had it a couple of months ago :)

[NoQ]

Yup, i hope it'll be comfy now.

[bogner]

The cmake and lit bits all look correct

[thopre]

Hi George,

This commit seems to be causing some testsuite regression on Armv8 (both Aarch64 and ARM) architectures:
http://lab.llvm.org:8011/builders/clang-cmake-armv8-full/builds/5450
http://lab.llvm.org:8011/builders/clang-cmake-armv8-quick/builds/5625

Best regards,

Thomas

[yroux]

Notice that the affected bots run two times the test cases where one execution passes and one fails, as you can see in (search Analysis/plist-macros.cpp for instance):

http://lab.llvm.org:8011/builders/clang-cmake-armv8-full/builds/5450/steps/ninja%20check%201/logs/stdio

and I think it is due to llvm-lit being invoked with "--param USE_Z3_SOLVER=0 --param USE_Z3_SOLVER=0"

[george.karpenkov]

Yes, investigating. Will rollback if not fixed in a few hours.

[george.karpenkov]

Should be fixed now.

test/Analysis/analyzer_test.py

 import lit.formats
 import lit.TestRunner
 
 # Custom format class for static analyzer tests
 class AnalyzerTest(lit.formats.ShTest):
 
+    def __init__(self, execute_external, use_z3_solver=False):
+        super(AnalyzerTest, self).__init__(execute_external)
+        self.use_z3_solver = use_z3_solver
+
     def execute(self, test, litConfig):
         results = []
 
         # Parse any test requirements ('REQUIRES: ')
         saved_test = test
         lit.TestRunner.parseIntegratedTestScript(test)
 
         if 'z3' not in test.requires:
             results.append(self.executeWithAnalyzeSubstitution(
                 saved_test, litConfig, '-analyzer-constraints=range'))
 
             if results[-1].code == lit.Test.FAIL:
                 return results[-1]
 
         # If z3 backend available, add an additional run line for it
-        if test.config.clang_staticanalyzer_z3 == '1':
+        if self.use_z3_solver == '1':
+            assert(test.config.clang_staticanalyzer_z3 == '1')
             results.append(self.executeWithAnalyzeSubstitution(
                 saved_test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3'))
 
         # Combine all result outputs into the last element
         for x in results:
             if x != results[-1]:
                 results[-1].output = x.output + results[-1].output
 

test/Analysis/lit.local.cfg

 # -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
 
 import site
 
 # Load the custom analyzer test format, which runs the test again with Z3 if it
 # is available.
 site.addsitedir(os.path.dirname(__file__))
 import analyzer_test
 config.test_format = analyzer_test.AnalyzerTest(
-        config.test_format.execute_external)
+        config.test_format.execute_external, config.use_z3_solver)
 
 if not config.root.clang_staticanalyzer:
     config.unsupported = True

test/CMakeLists.txt

 if (CLANG_ENABLE_STATIC_ANALYZER AND CLANG_BUILD_EXAMPLES)
   list(APPEND CLANG_TEST_DEPS
     SampleAnalyzerPlugin
     )
 endif ()
 
 set(CLANG_TEST_PARAMS
   clang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+  USE_Z3_SOLVER=0
   )
 
+set(ANALYZER_TEST_PARAMS
+  USE_Z3_SOLVER=0)
+
+set(ANALYZER_TEST_PARAMS_Z3
+  USE_Z3_SOLVER=1)
+
 if( NOT CLANG_BUILT_STANDALONE )
   list(APPEND CLANG_TEST_DEPS
     llvm-config
     FileCheck count not
     llc
     llvm-as
     llvm-bcanalyzer
     llvm-cat
add_lit_testsuite(check-clang "Running the Clang regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   #LIT ${LLVM_LIT}
   PARAMS ${CLANG_TEST_PARAMS}
   DEPENDS ${CLANG_TEST_DEPS}
   ARGS ${CLANG_TEST_EXTRA_ARGS}
   )
 set_target_properties(check-clang PROPERTIES FOLDER "Clang tests")
 
+if (CLANG_ENABLE_STATIC_ANALYZER)
+  add_lit_testsuite(check-clang-analyzer "Running the Clang analyzer tests"
+    ${CMAKE_CURRENT_BINARY_DIR}/Analysis
+    PARAMS ${ANALYZER_TEST_PARAMS}
+    DEPENDS ${CLANG_TEST_DEPS})
+  set_target_properties(check-clang-analyzer PROPERTIES FOLDER "Clang tests")
+
+
+  if (CLANG_ANALYZER_WITH_Z3)
+    add_lit_testsuite(check-clang-analyzer-z3 "Running the Clang analyzer tests, using Z3 as a solver"
+      ${CMAKE_CURRENT_BINARY_DIR}/Analysis
+      PARAMS ${ANALYZER_TEST_PARAMS_Z3}
+      DEPENDS ${CLANG_TEST_DEPS})
+    set_target_properties(check-clang-analyzer-z3 PROPERTIES FOLDER "Clang tests")
+  endif()
+
+endif()
+
 add_lit_testsuites(CLANG ${CMAKE_CURRENT_SOURCE_DIR}
   PARAMS ${CLANG_TEST_PARAMS}
   DEPENDS ${CLANG_TEST_DEPS}
 )
 
 # Add a legacy target spelling: clang-test
 add_custom_target(clang-test)
 add_dependencies(clang-test check-clang)

test/lit.site.cfg.py.in

 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
 config.clang_staticanalyzer_z3 = "@CLANG_ANALYZER_WITH_Z3@"
 config.clang_examples = @CLANG_BUILD_EXAMPLES@
 config.enable_shared = @ENABLE_SHARED@
 config.enable_backtrace = @ENABLE_BACKTRACES@
 config.host_arch = "@HOST_ARCH@"
 config.python_executable = "@PYTHON_EXECUTABLE@"
+config.use_z3_solver = "@USE_Z3_SOLVER@"
 
 # Support substitution of the tools and libs dirs with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
 try:
     config.clang_tools_dir = config.clang_tools_dir % lit_config.params
     config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
     config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
     config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.use_z3_solver = lit_config.params['USE_Z3_SOLVER']
 except KeyError:
     e = sys.exc_info()[1]
     key, = e.args
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 @LIT_SITE_CFG_IN_FOOTER@
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg.py")