[PGO] Add support to enable testing multiple supported targets

[davidxl]

Summary

This is the method used by sanitizer runtime as well. It allows testing of -m32 on 64 bit systems.

[samsonov]

LGTM

[vsk]

Hi David,

This commit enables tests on 64-bit Darwin which have apparently been failing. I now see:

******************** TEST 'Profile-i386 :: instrprof-basic.c' FAILED ********************
Script:
--
/Users/vk/Desktop/llvm/Debug+Asserts/./bin/clang -arch i386  -fprofile-instr-generate -o /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp -O3 /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-basic.c
env LLVM_PROFILE_FILE=/Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp.profraw  /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp
llvm-profdata merge -o /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp.profdata /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp.profraw
/Users/vk/Desktop/llvm/Debug+Asserts/./bin/clang -arch i386  -fprofile-instr-use=/Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp.profdata -o - -S -emit-llvm /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-basic.c | FileCheck /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-basic.c
--
Exit Code: 139

Command Output (stderr):
--
/Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.script: line 4: 20417 Segmentation fault: 11  env LLVM_PROFILE_FILE=/Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp.profraw /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-basic.c.tmp

The segmentation fault is in __llvm_profile_gather_value_data(). Accessing I->Values dereferences 0x1:

(lldb) p *I
(__llvm_profile_data) $11 = {
  NameRef = 10864
  FuncHash = 48378511633216
  CounterPtr = 0x00000000
  FunctionPointer = 0x0000df1e
  Values = 0x00000001
  NumCounters = 0
  NumValueSites = ([0] = 13723)
}

There's a second issue with corrupt raw profiles:

********************
FAIL: Profile-i386 :: instrprof-without-libc.c (39 of 96)
******************** TEST 'Profile-i386 :: instrprof-without-libc.c' FAILED ********************
Script:
--
/Users/vk/Desktop/llvm/Debug+Asserts/./bin/clang -arch i386  -fprofile-instr-generate -DCHECK_SYMBOLS -O3 -o /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.symbols /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-without-libc.c
llvm-nm /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.symbols | FileCheck /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-without-libc.c --check-prefix=CHECK-SYMBOLS
/Users/vk/Desktop/llvm/Debug+Asserts/./bin/clang -arch i386  -fprofile-instr-generate -O3 -o /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-without-libc.c
/Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.profraw
llvm-profdata merge -o /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.profdata /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.profraw
/Users/vk/Desktop/llvm/Debug+Asserts/./bin/clang -arch i386  -fprofile-instr-use=/Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.profdata -o - -S -emit-llvm /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-without-libc.c | FileCheck /Users/vk/Desktop/llvm/projects/compiler-rt/test/profile/instrprof-without-libc.c
--
Exit Code: 1

Command Output (stderr):
--
error: /Users/vk/Desktop/llvm/build/projects/compiler-rt/test/profile/Profile-i386/Output/instrprof-without-libc.c.tmp.profraw: Invalid instrumentation profile data (file header is corrupt)

This happens because ProfileSize = 34359738544 (pretty close to 32 GB, which seems wrong):

    frame #0: 0x00000001001fa851 llvm-profdata`llvm::RawInstrProfReader<unsigned int>::readHeader(this=0x0000000100b00000, Header=0x0000000100a00110) + 801 at InstrProfReader.cpp:337
   334 	
   335 	  auto *Start = reinterpret_cast<const char *>(&Header);
   336 	  if (Start + ProfileSize > DataBuffer->getBufferEnd())
-> 337 	    return error(instrprof_error::bad_header);

I don't know if this last issue is affected by this commit, but I'm not sure why these tests are unsupported (they don't seem Linux-specific):

UNSUPPORTED: Profile-x86_64 :: Linux/coverage_ctors.cpp (1 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/coverage_dtor.cpp (2 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/instrprof-basic.c (3 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/coverage_test.cpp (4 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/coverage_shared.test (5 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/instrprof-comdat.test (6 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/instrprof-dlopen.test (7 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/instrprof-dynamic-one-shared.test (8 of 32)
UNSUPPORTED: Profile-x86_64 :: Linux/instrprof-dynamic-two-shared.test (9 of 32)

For reference, I configure llvm with: -DCMAKE_CROSSCOMPILING=False -DLLVM_TARGET_ARCH="X86" -DLLVM_TARGETS_TO_BUILD="X86;AArch64;ARM" -DCMAKE_BUILD_TYPE:STRING=Debug -DLLVM_ENABLE_ASSERTIONS=True -DLLVM_INCLUDE_TESTS=True -DLLVM_ENABLE_WARNINGS=False -G 'Ninja' ../.

AFAICT (checking -###), the compiler is grabbing the correct copy of compiler-rt and it has the right slice:

$ ninja check-profile [...]
$ file "/Users/vk/Desktop/llvm/build/bin/../lib/clang/3.9.0/lib/darwin/libclang_rt.profile_osx.a"
/Users/vk/Desktop/llvm/build/bin/../lib/clang/3.9.0/lib/darwin/libclang_rt.profile_osx.a: Mach-O universal binary with 3 architectures: [i386: current ar archive random library] [x86_64: current ar archive random library] [x86_64h: current ar archive random library]
/Users/vk/Desktop/llvm/build/bin/../lib/clang/3.9.0/lib/darwin/libclang_rt.profile_osx.a (for architecture i386):	current ar archive random library
/Users/vk/Desktop/llvm/build/bin/../lib/clang/3.9.0/lib/darwin/libclang_rt.profile_osx.a (for architecture x86_64):	current ar archive random library
/Users/vk/Desktop/llvm/build/bin/../lib/clang/3.9.0/lib/darwin/libclang_rt.profile_osx.a (for architecture x86_64h):	current ar archive random library

I'll try to get to the bottom of this, but any help is appreciated.

[davidxl]

Is the root cause of the issue understood now?

thanks,

David

[davidxl]

Alexey, This seems to have regressed fairly recently. I just noticed that
check-profile only tests X86_64 target

This is because PROFILE_SUPPORTED_ARCH is not properly defined by
filter_available_targets(PROFILE_SUPPORTED_ARCH
${ALL_PROFILE_SUPPORTED_ARCH})

UBSAN_SUPPORTED_ARCH seems to have the same problem. When I do ninja
check-pubsan, only X86_64 target is tested.

Are you aware of any recent changes that are related?

thanks,

David

compiler-rt/trunk/test/profile/CMakeLists.txt

 set(PROFILE_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(PROFILE_LIT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 
+set(PROFILE_TESTSUITES)
 set(PROFILE_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 if(NOT COMPILER_RT_STANDALONE_BUILD)
   list(APPEND PROFILE_TEST_DEPS profile llvm-profdata)
 endif()
 
+set(PROFILE_TEST_ARCH ${PROFILE_SUPPORTED_ARCH})
+if(APPLE)
+  darwin_filter_host_archs(PROFILE_SUPPORTED_ARCH PROFILE_TEST_ARCH)
+endif()
+
+foreach(arch ${PROFILE_TEST_ARCH})
+  set(PROFILE_TEST_TARGET_ARCH ${arch})
+  if(${arch} MATCHES "arm|aarch64")
+    # This is only true if we're cross-compiling.
+    set(PROFILE_TEST_TARGET_CFLAGS ${COMPILER_RT_TEST_COMPILER_CFLAGS})
+  else()
+    get_target_flags_for_arch(${arch} PROFILE_TEST_TARGET_CFLAGS)
+    string(REPLACE ";" " " PROFILE_TEST_TARGET_CFLAGS "${PROFILE_TEST_TARGET_CFLAGS}")
+  endif()
+  set(CONFIG_NAME Profile-${arch})
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
-  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
    )
+  list(APPEND PROFILE_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
+endforeach()
+
 add_lit_testsuite(check-profile "Running the profile tests"
-  ${CMAKE_CURRENT_BINARY_DIR}
+  ${PROFILE_TESTSUITES}
   DEPENDS ${PROFILE_TEST_DEPS})
 set_target_properties(check-profile PROPERTIES FOLDER "Profile tests")

compiler-rt/trunk/test/profile/lit.cfg

 # -*- Python -*-
 
 import os
 
+def get_required_attr(config, attr_name):
+  attr_value = getattr(config, attr_name, None)
+  if attr_value == None:
+    lit_config.fatal(
+      "No attribute %r in test configuration! You may need to run "
+      "tests from your build directory or add this attribute "
+      "to lit.site.cfg " % attr_name)
+  return attr_value
+
 # Setup config name.
-config.name = 'Profile'
+config.name = 'Profile-' + config.target_arch
 
 # Setup source root.
 config.test_source_root = os.path.dirname(__file__)
 
 # Setup executable root.
 if hasattr(config, 'profile_lit_binary_dir') and \
         config.profile_lit_binary_dir is not None:
-    config.test_exec_root = config.profile_lit_binary_dir
+    config.test_exec_root = os.path.join(config.profile_lit_binary_dir, config.name)
 
 # If the above check didn't work, we're probably in the source tree.  Use some
 # magic to re-execute from the build tree.
 if config.test_exec_root is None:
     # The magic relies on knowing compilerrt_site_basedir.
     compilerrt_basedir = lit_config.params.get('compilerrt_site_basedir', None)
     if compilerrt_basedir:
         site_cfg = os.path.join(compilerrt_basedir, 'profile', 'lit.site.cfg')
         if os.path.exists(site_cfg):
             lit_config.load_config(config, site_cfg)
             raise SystemExit
 
 if config.host_os in ['Linux']:
   extra_linkflags = ["-ldl"]
 else:
   extra_linkflags = []
 
 # Test suffixes.
 config.suffixes = ['.c', '.cc', '.cpp', '.m', '.mm', '.ll', '.test']
 
 # What to exclude.
 config.excludes = ['Inputs']
 
 # Clang flags.
-clang_cflags = [config.target_cflags] + extra_linkflags
+target_cflags=[get_required_attr(config, "target_cflags")]
+clang_cflags = target_cflags + extra_linkflags
+clang_cxxflags = config.cxx_mode_flags + clang_cflags
 
 def build_invocation(compile_flags):
   return " " + " ".join([config.clang] + compile_flags) + " "
 
 # Add clang substitutions.
 config.substitutions.append( ("%clang ", build_invocation(clang_cflags)) )
+config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
 config.substitutions.append( ("%clang_profgen ", build_invocation(clang_cflags) + " -fprofile-instr-generate ") )
 config.substitutions.append( ("%clang_profuse=", build_invocation(clang_cflags) + " -fprofile-instr-use=") )
+config.substitutions.append( ("%clangxx_profgen ", build_invocation(clang_cxxflags) + " -fprofile-instr-generate ") )
+config.substitutions.append( ("%clangxx_profuse=", build_invocation(clang_cxxflags) + " -fprofile-instr-use=") )
 config.substitutions.append( ("%clang_profgen_gcc=", build_invocation(clang_cflags) + " -fprofile-generate=") )
 config.substitutions.append( ("%clang_profuse_gcc=", build_invocation(clang_cflags) + " -fprofile-use=") )
 
 if config.host_os not in ['Darwin', 'FreeBSD', 'Linux']:
   config.unsupported = True
 
 if config.target_arch in ['armv7l']:
   config.unsupported = True

compiler-rt/trunk/test/profile/lit.site.cfg.in

 ## Autogenerated by LLVM/Clang configuration.
 # Do not edit!
 
 # Tool-specific config options.
 config.profile_lit_binary_dir = "@PROFILE_LIT_BINARY_DIR@"
+config.target_cflags = "@PROFILE_TEST_TARGET_CFLAGS@"
+config.target_arch = "@PROFILE_TEST_TARGET_ARCH@"
 
 # Load common config for all compiler-rt lit tests.
 lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured")
 
 # Load tool-specific config that would do the real work.
 lit_config.load_config(config, "@PROFILE_LIT_SOURCE_DIR@/lit.cfg")