[clang-tools-extra] [cmake] Support running extra clang tool tests without static analyzer

[mgorny]

Summary

Support running the extra clang tool tests when the static analyzer
is disabled. Disable the relevant clang-tidy tests and one include-fixer
test that require it to work.

Previously, the tests were disabled entirely with
CLANG_ENABLE_STATIC_ANALYZER being false. Now, the tests are being
enabled and the relevant tests are excluded and marked unsupported
appropriately.

In order to disable clang-tidy tests, the whole test directory is added
to the exclude lists, to avoid having to explicitly add 'REQUIRES' line
to every single test. If the other solution is preferable, I can update
the patch.

The yamldb_plugin include-fixer test is also updated to be disabled
without static analyzer. It fails in that case because clang is not
outputting a replacement suggestion -- but I don't know the exact
reason why it does not do that.

[mgorny]

test/include-fixer/yamldb_plugin.cpp

7

Differential is showing this a bit weird, all that's changed here and below are adjusted line numbers.

[alexfh]

This could be done even more granularly, since clang-tidy only needs static analyzer to run clang-analyzer-* checks. But that would involve some careful #ifdef-ing of the parts of clang-tidy/ClangTidy.cpp, so I'll understand if you say you have little interest in this.

[alexfh]

IIUC, these are the only clang-tidy tests that need static analyzer:

clang/tools/extra/test/clang-tidy/nolint.cpp
clang/tools/extra/test/clang-tidy/static-analyzer.cpp
clang/tools/extra/test/clang-tidy/static-analyzer-config.cpp
clang/tools/extra/test/clang-tidy/temporaries.cpp

as well as the mpi module and its tests:
clang/tools/extra/clang-tidy/mpi/*
clang/tools/extra/test/clang-tidy/mpi-buffer-deref.cpp
clang/tools/extra/test/clang-tidy/mpi-type-mismatch.cpp

[mgorny]

I might be missing something but currently clang-tidy is not built at all when static analyzer is disabled.

[alexfh]

In D37188#854144, @mgorny wrote:

I might be missing something but currently clang-tidy is not built at all when static analyzer is disabled.

Yes, but it could be built except for the parts that actually need the static analyzer. As I've said, this would require careful surgery of ClangTidy.cpp and of some other files.

We can do that later.

LG

[mgorny]

Thanks for the clarification.

CMakeLists.txt

 add_subdirectory(clang-query)
 add_subdirectory(clang-move)
 add_subdirectory(clangd)
 add_subdirectory(include-fixer)
 add_subdirectory(pp-trace)
 add_subdirectory(tool-template)
 
 # Add the common testsuite after all the tools.
-# TODO: Support tests with more granularity when features are off?
-if(CLANG_ENABLE_STATIC_ANALYZER AND CLANG_INCLUDE_TESTS)
+if(CLANG_INCLUDE_TESTS)
 add_subdirectory(test)
 add_subdirectory(unittests)
 endif()
 
 option(CLANG_TOOLS_EXTRA_INCLUDE_DOCS "Generate build targets for the Clang Extra Tools docs."
   ${LLVM_INCLUDE_DOCS})
 if( CLANG_TOOLS_EXTRA_INCLUDE_DOCS )
   add_subdirectory(docs)
 endif()
 

test/CMakeLists.txt

 if (CMAKE_CFG_INTDIR STREQUAL ".")
   set(LLVM_BUILD_MODE ".")
 else ()
   set(LLVM_BUILD_MODE "%(build_mode)s")
 endif ()
 
 string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} CLANG_TOOLS_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
 
+llvm_canonicalize_cmake_booleans(
+  CLANG_ENABLE_STATIC_ANALYZER)
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
   )
 
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
   )
 
 option(CLANG_TOOLS_TEST_USE_VG "Run Clang tools' tests under Valgrind" OFF)
 if(CLANG_TOOLS_TEST_USE_VG)
   set(CLANG_TOOLS_TEST_EXTRA_ARGS ${CLANG_TEST_EXTRA_ARGS} "--vg")
 endif()
 
 set(CLANG_TOOLS_TEST_DEPS
-  # clang-tidy tests require it.
-  clang-headers
-
-  # For the clang-tidy libclang integration test.
-  c-index-test
   # For the clang-apply-replacements test that uses clang-rename.
   clang-rename
 
   # Individual tools we test.
   clang-apply-replacements
   clang-change-namespace
   clangd
   clang-include-fixer
   clang-move
   clang-query
   clang-reorder-fields
-  clang-tidy
   find-all-symbols
   modularize
   pp-trace
 
   # Unit tests
   ExtraToolsUnitTests
   )
 
+if(CLANG_ENABLE_STATIC_ANALYZER)
+  list(APPEND CLANG_TOOLS_TEST_DEPS
+    # For the clang-tidy libclang integration test.
+    c-index-test
+    # clang-tidy tests require it.
+    clang-headers
+
+    clang-tidy
+    )
+endif()
+
 set(llvm_utils
   FileCheck count not
   )
 
 foreach(t ${llvm_utils})
   if(TARGET ${t})
     list(APPEND CLANG_TOOLS_TEST_DEPS ${t})
   endif()

test/include-fixer/yamldb_plugin.cpp

+// REQUIRES: static-analyzer
 // RUN: c-index-test -test-load-source-reparse 2 all %s -Xclang -add-plugin -Xclang clang-include-fixer -fspell-checking -Xclang -plugin-arg-clang-include-fixer -Xclang -input=%p/Inputs/fake_yaml_db.yaml 2>&1 | FileCheck %s
 
 foo f;
 foo g;
 unknown u;
 
-// CHECK: yamldb_plugin.cpp:3:1: error: unknown type name 'foo'; did you mean 'foo'?
-// CHECK: Number FIX-ITs = 1
-// CHECK: FIX-IT: Replace [3:1 - 3:4] with "foo"
-// CHECK: yamldb_plugin.cpp:3:1: note: Add '#include "foo.h"' to provide the missing declaration [clang-include-fixer]
-// CHECK: Number FIX-ITs = 1
-// CHECK: FIX-IT: Insert "#include "foo.h"
 // CHECK: yamldb_plugin.cpp:4:1: error: unknown type name 'foo'; did you mean 'foo'?
 // CHECK: Number FIX-ITs = 1
 // CHECK: FIX-IT: Replace [4:1 - 4:4] with "foo"
 // CHECK: yamldb_plugin.cpp:4:1: note: Add '#include "foo.h"' to provide the missing declaration [clang-include-fixer]
 // CHECK: Number FIX-ITs = 1
 // CHECK: FIX-IT: Insert "#include "foo.h"
-// CHECK: " at 3:1
-// CHECK: yamldb_plugin.cpp:5:1:
+// CHECK: yamldb_plugin.cpp:5:1: error: unknown type name 'foo'; did you mean 'foo'?
+// CHECK: Number FIX-ITs = 1
+// CHECK: FIX-IT: Replace [5:1 - 5:4] with "foo"
+// CHECK: yamldb_plugin.cpp:5:1: note: Add '#include "foo.h"' to provide the missing declaration [clang-include-fixer]
+// CHECK: Number FIX-ITs = 1
+// CHECK: FIX-IT: Insert "#include "foo.h"
+// CHECK: " at 4:1
+// CHECK: yamldb_plugin.cpp:6:1:
 // CHECK: error: unknown type name 'unknown'
 // CHECK: Number FIX-ITs = 0
 // CHECK-NOT: error
 // CHECK-NOT: FIX-IT

test/lit.cfg

 # Exclude MSYS due to transforming '/' to 'X:/mingwroot/'.
 if not platform.system() in ['Windows'] or not execute_external:
     config.available_features.add('shell-preserves-root')
 
 # ANSI escape sequences in non-dumb terminal
 if platform.system() not in ['Windows']:
     config.available_features.add('ansi-escape-sequences')
 
+if config.clang_staticanalyzer:
+    config.available_features.add('static-analyzer')
     check_clang_tidy = os.path.join(
         config.test_source_root, "clang-tidy", "check_clang_tidy.py")
     config.substitutions.append(
         ('%check_clang_tidy',
          '%s %s' % (config.python_executable, check_clang_tidy)) )
     clang_tidy_diff = os.path.join(
         config.test_source_root, "..", "clang-tidy", "tool", "clang-tidy-diff.py")
     config.substitutions.append(
         ('%clang_tidy_diff',
          '%s %s' % (config.python_executable, clang_tidy_diff)) )
+else:
+    # exclude the clang-tidy test directory
+    config.excludes.append('clang-tidy')

test/lit.site.cfg.in

 @LIT_SITE_CFG_IN_HEADER@
 
 import sys
 
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
 config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
 config.clang_tools_binary_dir = "@CLANG_TOOLS_BINARY_DIR@"
 config.clang_tools_dir = "@CLANG_TOOLS_DIR@"
 config.clang_libs_dir = "@SHLIBDIR@"
 config.python_executable = "@PYTHON_EXECUTABLE@"
 config.target_triple = "@TARGET_TRIPLE@"
+config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
 
 # Support substitution of the tools and libs dirs with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
 try:
     config.clang_tools_dir = config.clang_tools_dir % lit_config.params
     config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
     config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
 except KeyError:
     e = sys.exc_info()[1]
     key, = e.args
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@CLANG_TOOLS_SOURCE_DIR@/test/lit.cfg")

unittests/CMakeLists.txt

 add_custom_target(ExtraToolsUnitTests)
 set_target_properties(ExtraToolsUnitTests PROPERTIES FOLDER "Extra Tools Unit Tests")
 
 function(add_extra_unittest test_dirname)
   add_unittest(ExtraToolsUnitTests ${test_dirname} ${ARGN})
 endfunction()
 
 add_subdirectory(change-namespace)
 add_subdirectory(clang-apply-replacements)
 add_subdirectory(clang-move)
 add_subdirectory(clang-query)
+if(CLANG_ENABLE_STATIC_ANALYZER)
   add_subdirectory(clang-tidy)
+endif()
 add_subdirectory(clangd)
 add_subdirectory(include-fixer)