Index: clang/test/Index/recover-bad-code-rdar_7487294.c =================================================================== --- clang/test/Index/recover-bad-code-rdar_7487294.c +++ clang/test/Index/recover-bad-code-rdar_7487294.c @@ -1,4 +1,4 @@ -// RUN: not %clang-cc1 -fsyntax-only %s 2>&1 | FileCheck %s +// RUN: not %clang_cc1 -fsyntax-only %s 2>&1 | FileCheck %s // IMPORTANT: This test case intentionally DOES NOT use --disable-free. It // tests that we are properly reclaiming the ASTs and we do not have a double free. Index: clang/test/lit.cfg.py =================================================================== --- clang/test/lit.cfg.py +++ clang/test/lit.cfg.py @@ -10,7 +10,8 @@ import lit.util from lit.llvm import llvm_config -from lit.llvm import ToolFilter +from lit.llvm.subst import ToolSubst +from lit.llvm.subst import FindTool # Configuration file for the 'lit' test runner. @@ -24,7 +25,8 @@ config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) # suffixes: A list of file extensions to treat as test files. -config.suffixes = ['.c', '.cpp', '.cppm', '.m', '.mm', '.cu', '.ll', '.cl', '.s', '.S', '.modulemap', '.test', '.rs'] +config.suffixes = ['.c', '.cpp', '.cppm', '.m', '.mm', '.cu', + '.ll', '.cl', '.s', '.S', '.modulemap', '.test', '.rs'] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent @@ -65,15 +67,21 @@ llvm_config.clear_environment(possibly_dangerous_env_vars) # Tweak the PATH to include the tools dir and the scripts dir. -llvm_config.with_environment('PATH', [config.llvm_tools_dir, config.clang_tools_dir], append_path=True) +llvm_config.with_environment( + 'PATH', [config.llvm_tools_dir, config.clang_tools_dir], append_path=True) -llvm_config.with_environment('LD_LIBRARY_PATH', [config.llvm_shlib_dir, config.llvm_libs_dir], append_path=True) +llvm_config.with_environment('LD_LIBRARY_PATH', [ + config.llvm_shlib_dir, config.llvm_libs_dir], append_path=True) # Propagate path to symbolizer for ASan/MSan. -llvm_config.with_system_environment(['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']) +llvm_config.with_system_environment( + ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']) + +llvm_config.use_default_substitutions() # Discover the 'clang' and 'clangcc' to use. + def inferClang(PATH): # Determine which clang to use. clang = os.getenv('CLANG') @@ -88,10 +96,11 @@ if not clang: lit_config.fatal("couldn't find 'clang' program, try setting " - "CLANG in your environment") + 'CLANG in your environment') return clang + config.clang = inferClang(config.environment['PATH']).replace('\\', '/') if not lit_config.quiet: lit_config.note('using clang: %r' % config.clang) @@ -106,95 +115,86 @@ if has_plugins and config.llvm_plugin_ext: config.available_features.add('plugins') -config.substitutions.append( ('%llvmshlibdir', config.llvm_shlib_dir) ) -config.substitutions.append( ('%pluginext', config.llvm_plugin_ext) ) -config.substitutions.append( ('%PATH%', config.environment['PATH']) ) +config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir)) +config.substitutions.append(('%pluginext', config.llvm_plugin_ext)) +config.substitutions.append(('%PATH%', config.environment['PATH'])) if config.clang_examples: config.available_features.add('examples') builtin_include_dir = llvm_config.get_clang_builtin_include_dir(config.clang) -config.substitutions.append( ('%clang_analyze_cc1', - '%clang_cc1 -analyze %analyze') ) -config.substitutions.append( ('%clang_cc1', - '%s -cc1 -internal-isystem %s -nostdsysteminc' - % (config.clang, builtin_include_dir)) ) -config.substitutions.append( ('%clang_cpp', ' ' + config.clang + - ' --driver-mode=cpp ')) -config.substitutions.append( ('%clang_cl', ' ' + config.clang + - ' --driver-mode=cl ')) -config.substitutions.append( ('%clangxx', ' ' + config.clang + - ' --driver-mode=g++ ')) - -clang_func_map = lit.util.which('clang-func-mapping', config.environment['PATH']) -if clang_func_map: - config.substitutions.append( ('%clang_func_map', ' ' + clang_func_map + ' ') ) - -config.substitutions.append( ('%clang', ' ' + config.clang + ' ') ) -config.substitutions.append( ('%test_debuginfo', - ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') ) -config.substitutions.append( ('%itanium_abi_triple', - llvm_config.make_itanium_abi_triple(config.target_triple)) ) -config.substitutions.append( ('%ms_abi_triple', - llvm_config.make_msabi_triple(config.target_triple)) ) -config.substitutions.append( ('%resource_dir', builtin_include_dir) ) -config.substitutions.append( ('%python', config.python_executable) ) -# The host triple might not be set, at least if we're compiling clang from -# an already installed llvm. -if config.host_triple and config.host_triple != '@LLVM_HOST_TRIPLE@': - config.substitutions.append( ('%target_itanium_abi_host_triple', - '--target=%s' % llvm_config.make_itanium_abi_triple(config.host_triple)) ) -else: - config.substitutions.append( ('%target_itanium_abi_host_triple', '') ) +tools = [ + # By specifying %clang_cc1 as part of the substitution, this substitution + # relies on repeated substitution, so must come before %clang_cc1. + ToolSubst('%clang_analyze_cc1', command='%clang_cc1', + extra_args=['-analyze', '%analyze']), + ToolSubst('%clang_cc1', command=config.clang, extra_args=[ + '-cc1', '-internal-isystem', builtin_include_dir, '-nostdsysteminc']), + ToolSubst('%clang_cpp', command=config.clang, + extra_args=['--driver-mode=cpp']), + ToolSubst('%clang_cl', command=config.clang, + extra_args=['--driver-mode=cl']), + ToolSubst('%clangxx', command=config.clang, + extra_args=['--driver-mode=g++']), + ToolSubst('%clang_func_map', command=FindTool( + 'clang-func-mapping'), unresolved='ignore'), + ToolSubst('%clang', command=config.clang), + ToolSubst('%test_debuginfo', command=os.path.join( + config.llvm_src_root, 'utils', 'test_debuginfo.pl')), + 'c-index-test', 'clang-check', 'clang-diff', 'clang-format', 'opt'] + +if config.clang_examples: + tools.append('clang-interpreter') + +# For each occurrence of a clang tool name, replace it with the full path to +# the build directory holding that tool. We explicitly specify the directories +# to search to ensure that we get the tools just built and not some random +# tools that might happen to be in the user's PATH. +tool_dirs = [config.clang_tools_dir, config.llvm_tools_dir] -config.substitutions.append( ('%src_include_dir', config.clang_src_dir + '/include') ) +llvm_config.add_tool_substitutions(tools, tool_dirs) # FIXME: Find nicer way to prohibit this. config.substitutions.append( - (' clang ', """*** Do not use 'clang' in tests, use '%clang'. ***""") ) + (' clang ', """*** Do not use 'clang' in tests, use '%clang'. ***""")) config.substitutions.append( (' clang\+\+ ', """*** Do not use 'clang++' in tests, use '%clangxx'. ***""")) config.substitutions.append( (' clang-cc ', - """*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***""") ) + """*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***""")) config.substitutions.append( (' clang -cc1 -analyze ', - """*** Do not use 'clang -cc1 -analyze' in tests, use '%clang_analyze_cc1'. ***""") ) + """*** Do not use 'clang -cc1 -analyze' in tests, use '%clang_analyze_cc1'. ***""")) config.substitutions.append( (' clang -cc1 ', - """*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***""") ) + """*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***""")) config.substitutions.append( (' %clang-cc1 ', - """*** invalid substitution, use '%clang_cc1'. ***""") ) + """*** invalid substitution, use '%clang_cc1'. ***""")) config.substitutions.append( (' %clang-cpp ', - """*** invalid substitution, use '%clang_cpp'. ***""") ) + """*** invalid substitution, use '%clang_cpp'. ***""")) config.substitutions.append( (' %clang-cl ', - """*** invalid substitution, use '%clang_cl'. ***""") ) - -# For each occurrence of a clang tool name, replace it with the full path to -# the build directory holding that tool. We explicitly specify the directories -# to search to ensure that we get the tools just built and not some random -# tools that might happen to be in the user's PATH. -tool_dirs = [config.clang_tools_dir, config.llvm_tools_dir] + """*** invalid substitution, use '%clang_cl'. ***""")) -tool_patterns = [ - 'FileCheck', 'c-index-test', - ToolFilter('clang-check', pre='-.', post='-.'), - ToolFilter('clang-diff', pre='-.', post='-.'), - ToolFilter('clang-format', pre='-.', post='-.'), - # FIXME: Some clang test uses opt? - ToolFilter('opt', pre='-.', post=r'/\-.'), - # Handle these specially as they are strings searched for during testing. - ToolFilter(r'\| \bcount\b', verbatim=True), - ToolFilter(r'\| \bnot\b', verbatim=True)] +config.substitutions.append(('%itanium_abi_triple', + llvm_config.make_itanium_abi_triple(config.target_triple))) +config.substitutions.append(('%ms_abi_triple', + llvm_config.make_msabi_triple(config.target_triple))) +config.substitutions.append(('%resource_dir', builtin_include_dir)) -if config.clang_examples: - tool_patterns.append(ToolFilter('clang-interpreter', '-.', '-.')) +# The host triple might not be set, at least if we're compiling clang from +# an already installed llvm. +if config.host_triple and config.host_triple != '@LLVM_HOST_TRIPLE@': + config.substitutions.append(('%target_itanium_abi_host_triple', + '--target=%s' % llvm_config.make_itanium_abi_triple(config.host_triple))) +else: + config.substitutions.append(('%target_itanium_abi_host_triple', '')) -llvm_config.add_tool_substitutions(tool_patterns, tool_dirs) +config.substitutions.append( + ('%src_include_dir', config.clang_src_dir + '/include')) # Set available features we allow tests to conditionalize on. # @@ -203,10 +203,10 @@ # Enabled/disabled features if config.clang_staticanalyzer: - config.available_features.add("staticanalyzer") + config.available_features.add('staticanalyzer') if config.clang_staticanalyzer_z3 == '1': - config.available_features.add("z3") + config.available_features.add('z3') # As of 2011.08, crash-recovery tests still do not pass on FreeBSD. if platform.system() not in ['FreeBSD']: @@ -227,22 +227,26 @@ config.available_features.add('libgcc') # Case-insensitive file system + + def is_filesystem_case_insensitive(): - handle, path = tempfile.mkstemp(prefix='case-test', dir=config.test_exec_root) + handle, path = tempfile.mkstemp( + prefix='case-test', dir=config.test_exec_root) isInsensitive = os.path.exists( os.path.join( os.path.dirname(path), os.path.basename(path).upper() - )) + )) os.close(handle) os.remove(path) return isInsensitive + if is_filesystem_case_insensitive(): config.available_features.add('case-insensitive-filesystem') # Tests that require the /dev/fd filesystem. -if os.path.exists("/dev/fd/0") and sys.platform not in ['cygwin']: +if os.path.exists('/dev/fd/0') and sys.platform not in ['cygwin']: config.available_features.add('dev-fd-fs') # Not set on native MS environment. @@ -266,28 +270,30 @@ if platform.system() not in ['Windows']: config.available_features.add('can-remove-opened-file') + def calculate_arch_features(arch_string): features = [] for arch in arch_string.split(): features.append(arch.lower() + '-registered-target') return features + llvm_config.feature_config( - [('--assertion-mode', {'ON' : 'asserts'}), - ('--cxxflags', {r'-D_GLIBCXX_DEBUG\b' : 'libstdcxx-safe-mode'}), - ('--targets-built', calculate_arch_features) - ]) + [('--assertion-mode', {'ON': 'asserts'}), + ('--cxxflags', {r'-D_GLIBCXX_DEBUG\b': 'libstdcxx-safe-mode'}), + ('--targets-built', calculate_arch_features) + ]) if lit.util.which('xmllint'): config.available_features.add('xmllint') if config.enable_backtrace: - config.available_features.add("backtrace") + config.available_features.add('backtrace') # Check if we should allow outputs to console. run_console_tests = int(lit_config.params.get('enable_console', '0')) if run_console_tests != 0: - config.available_features.add('console') + config.available_features.add('console') lit.util.usePlatformSdkOnDarwin(config, lit_config) macOSSDKVersion = lit.util.findPlatformSdkVersionOnMacOS(config, lit_config) Index: lld/test/lit.cfg.py =================================================================== --- lld/test/lit.cfg.py +++ lld/test/lit.cfg.py @@ -10,7 +10,7 @@ import lit.util from lit.llvm import llvm_config -from lit.llvm import ToolFilter +from lit.llvm.subst import ToolSubst # Configuration file for the 'lit' test runner. @@ -43,24 +43,22 @@ llvm_config.with_environment('LD_LIBRARY_PATH', [config.lld_libs_dir, config.llvm_libs_dir], append_path=True) +llvm_config.use_default_substitutions() + # For each occurrence of a clang tool name, replace it with the full path to # the build directory holding that tool. We explicitly specify the directories # to search to ensure that we get the tools just built and not some random # tools that might happen to be in the user's PATH. tool_dirs = [config.lld_tools_dir, config.llvm_tools_dir] -config.substitutions.append( (r"\bld.lld\b", 'ld.lld --full-shutdown') ) - tool_patterns = [ - 'FileCheck', 'not', 'ld.lld', 'lld-link', 'llvm-as', 'llvm-mc', 'llvm-nm', + ToolSubst('ld.lld', extra_args=['--full-shutdown']), + 'lld-link', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump', 'llvm-pdbutil', 'llvm-readobj', 'obj2yaml', 'yaml2obj', - ToolFilter('lld', pre='-./', post='-.')] + 'lld'] llvm_config.add_tool_substitutions(tool_patterns, tool_dirs) -# Add site-specific substitutions. -config.substitutions.append( ('%python', config.python_executable) ) - # When running under valgrind, we mangle '-vg' onto the end of the triple so we # can check it with XFAIL and XTARGET. if lit_config.useValgrind: @@ -75,17 +73,17 @@ config.available_features.add('demangler') llvm_config.feature_config( - [('--build-mode', {'DEBUG' : 'debug'}), - ('--assertion-mode', {'ON' : 'asserts'}), - ('--targets-built', {'AArch64' : 'aarch64', - 'AMDGPU' : 'amdgpu', - 'ARM' : 'arm', - 'AVR' : 'avr', - 'Mips' : 'mips', - 'PowerPC' : 'ppc', - 'Sparc' : 'sparc', - 'X86' : 'x86'}) - ]) + [('--build-mode', {'DEBUG': 'debug'}), + ('--assertion-mode', {'ON': 'asserts'}), + ('--targets-built', {'AArch64': 'aarch64', + 'AMDGPU': 'amdgpu', + 'ARM': 'arm', + 'AVR': 'avr', + 'Mips': 'mips', + 'PowerPC': 'ppc', + 'Sparc': 'sparc', + 'X86': 'x86'}) + ]) # Set a fake constant version so that we get consitent output. config.environment['LLD_VERSION'] = 'LLD 1.0' @@ -94,8 +92,8 @@ # cvtres, which always accompanies it. Alternatively, check if we can use # libxml2 to merge manifests. if (lit.util.which('cvtres', config.environment['PATH'])) or \ - (config.llvm_libxml2_enabled == "1"): + (config.llvm_libxml2_enabled == '1'): config.available_features.add('manifest_tool') -if (config.llvm_libxml2_enabled == "1"): +if (config.llvm_libxml2_enabled == '1'): config.available_features.add('libxml2') Index: llvm/test/lit.cfg.py =================================================================== --- llvm/test/lit.cfg.py +++ llvm/test/lit.cfg.py @@ -11,7 +11,8 @@ import lit.util import lit.formats from lit.llvm import llvm_config -from lit.llvm import ToolFilter +from lit.llvm.subst import FindTool +from lit.llvm.subst import ToolSubst # name: The name of this test suite. config.name = 'LLVM' @@ -38,7 +39,8 @@ llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True) # Propagate some variables from the host environment. -llvm_config.with_system_environment(['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']) +llvm_config.with_system_environment( + ['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']) # Set up OCAMLPATH to include newly built OCaml libraries. @@ -50,57 +52,61 @@ llvm_config.with_environment('OCAMLPATH', llvm_ocaml_lib, append_path=True) llvm_config.with_system_environment('CAML_LD_LIBRARY_PATH') -llvm_config.with_environment('CAML_LD_LIBRARY_PATH', llvm_ocaml_lib, append_path=True) +llvm_config.with_environment( + 'CAML_LD_LIBRARY_PATH', llvm_ocaml_lib, append_path=True) # Set up OCAMLRUNPARAM to enable backtraces in OCaml tests. llvm_config.with_environment('OCAMLRUNPARAM', 'b') # Provide the path to asan runtime lib 'libclang_rt.asan_osx_dynamic.dylib' if # available. This is darwin specific since it's currently only needed on darwin. + + def get_asan_rtlib(): - if not "Address" in config.llvm_use_sanitizer or \ - not "Darwin" in config.host_os or \ - not "x86" in config.host_triple: - return "" + if not 'Address' in config.llvm_use_sanitizer or \ + not 'Darwin' in config.host_os or \ + not 'x86' in config.host_triple: + return '' try: import glob except: - print("glob module not found, skipping get_asan_rtlib() lookup") - return "" + print('glob module not found, skipping get_asan_rtlib() lookup') + return '' # The libclang_rt.asan_osx_dynamic.dylib path is obtained using the relative # path from the host cc. - host_lib_dir = os.path.join(os.path.dirname(config.host_cc), "../lib") + host_lib_dir = os.path.join(os.path.dirname(config.host_cc), '../lib') asan_dylib_dir_pattern = host_lib_dir + \ - "/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" + '/clang/*/lib/darwin/libclang_rt.asan_osx_dynamic.dylib' found_dylibs = glob.glob(asan_dylib_dir_pattern) if len(found_dylibs) != 1: - return "" + return '' return found_dylibs[0] -lli = 'lli' + +llvm_config.use_default_substitutions() + +# Add site-specific substitutions. +config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir)) +config.substitutions.append(('%shlibext', config.llvm_shlib_ext)) +config.substitutions.append(('%exeext', config.llvm_exe_ext)) +config.substitutions.append(('%host_cc', config.host_cc)) + + +lli_args = [] # The target triple used by default by lli is the process target triple (some # triple appropriate for generating code for the current process) but because # we don't support COFF in MCJIT well enough for the tests, force ELF format on # Windows. FIXME: the process target triple should be used here, but this is # difficult to obtain on Windows. if re.search(r'cygwin|mingw32|windows-gnu|windows-msvc|win32', config.host_triple): - lli += ' -mtriple='+config.host_triple+'-elf' -config.substitutions.append( ('%lli', lli ) ) + lli_args = ['-mtriple=' + config.host_triple + '-elf'] + +llc_args = [] # Similarly, have a macro to use llc with DWARF even when the host is win32. -llc_dwarf = 'llc' if re.search(r'win32', config.target_triple): - llc_dwarf += ' -mtriple='+config.target_triple.replace('-win32', '-mingw32') -config.substitutions.append( ('%llc_dwarf', llc_dwarf) ) - -# Add site-specific substitutions. -config.substitutions.append( ('%gold', config.gold_executable) ) -config.substitutions.append( ('%go', config.go_executable) ) -config.substitutions.append( ('%llvmshlibdir', config.llvm_shlib_dir) ) -config.substitutions.append( ('%shlibext', config.llvm_shlib_ext) ) -config.substitutions.append( ('%exeext', config.llvm_exe_ext) ) -config.substitutions.append( ('%python', config.python_executable) ) -config.substitutions.append( ('%host_cc', config.host_cc) ) + llc_args = [' -mtriple=' + + config.target_triple.replace('-win32', '-mingw32')] # Provide the path to asan runtime lib if available. On darwin, this lib needs # to be loaded via DYLD_INSERT_LIBRARIES before libLTO.dylib in case the files @@ -108,36 +114,28 @@ ld64_cmd = config.ld64_executable asan_rtlib = get_asan_rtlib() if asan_rtlib: - ld64_cmd = "DYLD_INSERT_LIBRARIES={} {}".format(asan_rtlib, ld64_cmd) -config.substitutions.append( ('%ld64', ld64_cmd) ) - -# OCaml substitutions. -# Support tests for both native and bytecode builds. -config.substitutions.append( ('%ocamlc', - "%s ocamlc -cclib -L%s %s" % - (config.ocamlfind_executable, config.llvm_lib_dir, config.ocaml_flags)) ) + ld64_cmd = 'DYLD_INSERT_LIBRARIES={} {}'.format(asan_rtlib, ld64_cmd) + +ocamlc_command = '%s ocamlc -cclib -L%s %s' % ( + config.ocamlfind_executable, config.llvm_lib_dir, config.ocaml_flags) +ocamlopt_command = 'true' if config.have_ocamlopt: - config.substitutions.append( ('%ocamlopt', - "%s ocamlopt -cclib -L%s -cclib -Wl,-rpath,%s %s" % - (config.ocamlfind_executable, config.llvm_lib_dir, config.llvm_lib_dir, config.ocaml_flags)) ) -else: - config.substitutions.append( ('%ocamlopt', "true" ) ) - -# For each occurrence of an llvm tool name as its own word, replace it -# with the full path to the build directory holding that tool. This -# ensures that we are testing the tools just built and not some random -# tools that might happen to be in the user's PATH. Thus this list -# includes every tool placed in $(LLVM_OBJ_ROOT)/$(BuildMode)/bin -# (llvm_tools_dir in lit parlance). - -# Avoid matching RUN line fragments that are actually part of -# path names or options or whatever. -# The regex is a pre-assertion to avoid matching a preceding -# dot, hyphen, carat, or slash (.foo, -foo, etc.). Some patterns -# also have a post-assertion to not match a trailing hyphen (foo-). -JUNKCHARS = r".-^/<" - -required_tools = [ + ocamlopt_command = '%s ocamlopt -cclib -L%s -cclib -Wl,-rpath,%s %s' % ( + config.ocamlfind_executable, config.llvm_lib_dir, config.llvm_lib_dir, config.ocaml_flags) + + +tools = [ + ToolSubst('%lli', FindTool('lli'), extra_args=lli_args), + ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args), + ToolSubst('%go', config.go_executable, unresolved='ignore'), + ToolSubst('%gold', config.gold_executable, unresolved='ignore'), + ToolSubst('%ld64', ld64_cmd, unresolved='ignore'), + ToolSubst('%ocamlc', ocamlc_command, unresolved='ignore'), + ToolSubst('%ocamlopt', ocamlopt_command, unresolved='ignore'), +] + +# FIXME: Why do we have both `lli` and `%lli` that do slightly different things? +tools.extend([ 'lli', 'llvm-ar', 'llvm-as', 'llvm-bcanalyzer', 'llvm-config', 'llvm-cov', 'llvm-cxxdump', 'llvm-cvtres', 'llvm-diff', 'llvm-dis', 'llvm-dsymutil', 'llvm-dwarfdump', 'llvm-extract', 'llvm-isel-fuzzer', 'llvm-lib', @@ -146,37 +144,30 @@ 'llvm-pdbutil', 'llvm-profdata', 'llvm-ranlib', 'llvm-readobj', 'llvm-rtdyld', 'llvm-size', 'llvm-split', 'llvm-strings', 'llvm-tblgen', 'llvm-c-test', 'llvm-cxxfilt', 'llvm-xray', 'yaml2obj', 'obj2yaml', - 'FileCheck', 'yaml-bench', 'verify-uselistorder', - ToolFilter('bugpoint', post='-'), - ToolFilter('llc', pre=JUNKCHARS), - ToolFilter('llvm-symbolizer', pre=JUNKCHARS), - ToolFilter('opt', JUNKCHARS), - ToolFilter('sancov', pre=JUNKCHARS), - ToolFilter('sanstats', pre=JUNKCHARS), - # Handle these specially as they are strings searched for during testing. - ToolFilter(r'\| \bcount\b', verbatim=True), - ToolFilter(r'\| \bnot\b', verbatim=True)] - -llvm_config.add_tool_substitutions(required_tools, config.llvm_tools_dir) - -# For tools that are optional depending on the config, we won't warn -# if they're missing. - -optional_tools = [ - 'llvm-go', 'llvm-mt', 'Kaleidoscope-Ch3', 'Kaleidoscope-Ch4', - 'Kaleidoscope-Ch5', 'Kaleidoscope-Ch6', 'Kaleidoscope-Ch7', - 'Kaleidoscope-Ch8'] -llvm_config.add_tool_substitutions(optional_tools, config.llvm_tools_dir, - warn_missing=False) - -### Targets + 'yaml-bench', 'verify-uselistorder', + 'bugpoint', 'llc', 'llvm-symbolizer', 'opt', 'sancov', 'sanstats']) + +# The following tools are optional +tools.extend([ + ToolSubst('llvm-go', unresolved='ignore'), + ToolSubst('llvm-mt', unresolved='ignore'), + ToolSubst('Kaleidoscope-Ch3', unresolved='ignore'), + ToolSubst('Kaleidoscope-Ch4', unresolved='ignore'), + ToolSubst('Kaleidoscope-Ch5', unresolved='ignore'), + ToolSubst('Kaleidoscope-Ch6', unresolved='ignore'), + ToolSubst('Kaleidoscope-Ch7', unresolved='ignore'), + ToolSubst('Kaleidoscope-Ch8', unresolved='ignore')]) + +llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir) + +# Targets config.targets = frozenset(config.targets_to_build.split()) for arch in config.targets_to_build.split(): config.available_features.add(arch.lower() + '-registered-target') -### Features +# Features # Others/can-execute.txt if sys.platform not in ['win32']: @@ -195,24 +186,26 @@ # Static libraries are not built if BUILD_SHARED_LIBS is ON. if not config.build_shared_libs: - config.available_features.add("static-libs") + config.available_features.add('static-libs') # Direct object generation if not 'hexagon' in config.target_triple: - config.available_features.add("object-emission") + config.available_features.add('object-emission') # LLVM can be configured with an empty default triple # Some tests are "generic" and require a valid default triple if config.target_triple: - config.available_features.add("default_triple") + config.available_features.add('default_triple') import subprocess + def have_ld_plugin_support(): if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'LLVMgold.so')): return False - ld_cmd = subprocess.Popen([config.gold_executable, '--help'], stdout = subprocess.PIPE, env={'LANG': 'C'}) + ld_cmd = subprocess.Popen( + [config.gold_executable, '--help'], stdout=subprocess.PIPE, env={'LANG': 'C'}) ld_out = ld_cmd.stdout.read().decode() ld_cmd.wait() @@ -233,21 +226,25 @@ if 'elf32ppc' in emulations: config.available_features.add('ld_emu_elf32ppc') - ld_version = subprocess.Popen([config.gold_executable, '--version'], stdout = subprocess.PIPE, env={'LANG': 'C'}) + ld_version = subprocess.Popen( + [config.gold_executable, '--version'], stdout=subprocess.PIPE, env={'LANG': 'C'}) if not 'GNU gold' in ld_version.stdout.read().decode(): return False ld_version.wait() return True + if have_ld_plugin_support(): config.available_features.add('ld_plugin') + def have_ld64_plugin_support(): if not config.llvm_tool_lto_build or config.ld64_executable == '': return False - ld_cmd = subprocess.Popen([config.ld64_executable, '-v'], stderr = subprocess.PIPE) + ld_cmd = subprocess.Popen( + [config.ld64_executable, '-v'], stderr=subprocess.PIPE) ld_out = ld_cmd.stderr.read().decode() ld_cmd.wait() @@ -256,22 +253,23 @@ return True + if have_ld64_plugin_support(): config.available_features.add('ld64_plugin') # Ask llvm-config about asserts and global-isel. llvm_config.feature_config( - [('--assertion-mode', {'ON' : 'asserts'}), - ('--has-global-isel', {'ON' : 'global-isel'})]) + [('--assertion-mode', {'ON': 'asserts'}), + ('--has-global-isel', {'ON': 'global-isel'})]) if 'darwin' == sys.platform: try: sysctl_cmd = subprocess.Popen(['sysctl', 'hw.optional.fma'], - stdout = subprocess.PIPE) + stdout=subprocess.PIPE) except OSError: - print("Could not exec sysctl") + print('Could not exec sysctl') result = sysctl_cmd.stdout.read().decode('ascii') - if -1 != result.find("hw.optional.fma: 1"): + if -1 != result.find('hw.optional.fma: 1'): config.available_features.add('fma3') sysctl_cmd.wait() @@ -282,5 +280,5 @@ if config.have_libxar: config.available_features.add('xar') -if config.llvm_libxml2_enabled == "1": +if config.llvm_libxml2_enabled == '1': config.available_features.add('libxml2') Index: llvm/utils/lit/lit/llvm/__init__.py =================================================================== --- llvm/utils/lit/lit/llvm/__init__.py +++ llvm/utils/lit/lit/llvm/__init__.py @@ -1,51 +1,9 @@ from lit.llvm import config -import lit.util -import re llvm_config = None -class ToolFilter(object): - """ - String-like class used to build regex substitution patterns for - llvm tools. Handles things like adding word-boundary patterns, - and filtering characters from the beginning an end of a tool name - """ - - def __init__(self, name, pre=None, post=None, verbatim=False): - """ - Construct a ToolFilter. - - name: the literal name of the substitution to look for. - - pre: If specified, the substitution will not find matches where - the character immediately preceding the word-boundary that begins - `name` is any of the characters in the string `pre`. - - post: If specified, the substitution will not find matches where - the character immediately after the word-boundary that ends `name` - is any of the characters specified in the string `post`. - - verbatim: If True, `name` is an exact regex that is passed to the - underlying substitution - """ - if verbatim: - self.regex = name - return - - def not_in(chars, where=''): - if not chars: - return '' - pattern_str = '|'.join(re.escape(x) for x in chars) - return r'(?{}!({}))'.format(where, pattern_str) - - self.regex = not_in(pre, '<') + r'\b' + name + r'\b' + not_in(post) - - def __str__(self): - return self.regex - def initialize(lit_config, test_config): global llvm_config llvm_config = config.LLVMConfig(lit_config, test_config) - Index: llvm/utils/lit/lit/llvm/config.py =================================================================== --- llvm/utils/lit/lit/llvm/config.py +++ llvm/utils/lit/lit/llvm/config.py @@ -5,10 +5,14 @@ import sys import lit.util +from lit.llvm.subst import FindTool +from lit.llvm.subst import ToolSubst + def binary_feature(on, feature, off_prefix): return feature if on else off_prefix + feature + class LLVMConfig(object): def __init__(self, lit_config, config): @@ -25,22 +29,21 @@ # Seek sane tools in directories and set to $PATH. path = self.lit_config.getToolsPath(config.lit_tools_dir, - config.environment['PATH'], - ['cmp.exe', 'grep.exe', 'sed.exe']) + config.environment['PATH'], + ['cmp.exe', 'grep.exe', 'sed.exe']) if path is not None: self.with_environment('PATH', path, append_path=True) self.use_lit_shell = True # Choose between lit's internal shell pipeline runner and a real shell. If # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override. - lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL") + lit_shell_env = os.environ.get('LIT_USE_INTERNAL_SHELL') if lit_shell_env: self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env) if not self.use_lit_shell: features.add('shell') - # Running on Darwin OS if platform.system() in ['Darwin']: # FIXME: lld uses the first, other projects use the second. @@ -57,29 +60,31 @@ host_triple = getattr(config, 'host_triple', None) target_triple = getattr(config, 'target_triple', None) if host_triple and host_triple == target_triple: - features.add("native") + features.add('native') # Sanitizers. sanitizers = getattr(config, 'llvm_use_sanitizer', '') sanitizers = frozenset(x.lower() for x in sanitizers.split(';')) features.add(binary_feature('address' in sanitizers, 'asan', 'not_')) features.add(binary_feature('memory' in sanitizers, 'msan', 'not_')) - features.add(binary_feature('undefined' in sanitizers, 'ubsan', 'not_')) + features.add(binary_feature( + 'undefined' in sanitizers, 'ubsan', 'not_')) have_zlib = getattr(config, 'have_zlib', None) features.add(binary_feature(have_zlib, 'zlib', 'no')) # Check if we should run long running tests. - long_tests = lit_config.params.get("run_long_tests", None) + long_tests = lit_config.params.get('run_long_tests', None) if lit.util.pythonize_bool(long_tests): - features.add("long_tests") + features.add('long_tests') if target_triple: if re.match(r'^x86_64.*-apple', target_triple): if 'address' in sanitizers: - self.with_environment('ASAN_OPTIONS', 'detect_leaks=1', append_path=True) + self.with_environment( + 'ASAN_OPTIONS', 'detect_leaks=1', append_path=True) if re.match(r'^x86_64.*-linux', target_triple): - features.add("x86_64-linux") + features.add('x86_64-linux') if re.match(r'.*-win32$', target_triple): features.add('target-windows') @@ -90,13 +95,14 @@ gmalloc_path_str = lit_config.params.get('gmalloc_path', '/usr/lib/libgmalloc.dylib') if gmalloc_path_str is not None: - self.with_environment('DYLD_INSERT_LIBRARIES', gmalloc_path_str) + self.with_environment( + 'DYLD_INSERT_LIBRARIES', gmalloc_path_str) breaking_checks = getattr(config, 'enable_abi_breaking_checks', None) if lit.util.pythonize_bool(breaking_checks): features.add('abi-breaking-checks') - def with_environment(self, variable, value, append_path = False): + def with_environment(self, variable, value, append_path=False): if append_path: # For paths, we should be able to take a list of them and process all # of them. @@ -129,8 +135,7 @@ value = os.pathsep.join(paths) self.config.environment[variable] = value - - def with_system_environment(self, variables, append_path = False): + def with_system_environment(self, variables, append_path=False): if lit.util.is_string(variables): variables = [variables] for v in variables: @@ -153,7 +158,7 @@ stderr = lit.util.to_string(stderr) return (stdout, stderr) except OSError: - self.lit_config.fatal("Could not run process %s" % command) + self.lit_config.fatal('Could not run process %s' % command) def feature_config(self, features): # Ask llvm-config about the specified feature. @@ -175,17 +180,18 @@ if re.search(re_pattern, feature_line): self.config.available_features.add(feature) - # Note that when substituting %clang_cc1 also fill in the include directory of # the builtin headers. Those are part of even a freestanding environment, but # Clang relies on the driver to locate them. def get_clang_builtin_include_dir(self, clang): # FIXME: Rather than just getting the version, we should have clang print # out its resource dir here in an easy to scrape form. - clang_dir, _ = self.get_process_output([clang, '-print-file-name=include']) + clang_dir, _ = self.get_process_output( + [clang, '-print-file-name=include']) if not clang_dir: - self.lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang) + self.lit_config.fatal( + "Couldn't find the include dir for Clang ('%s')" % clang) clang_dir = clang_dir.strip() if sys.platform in ['win32'] and not self.use_lit_shell: @@ -197,62 +203,71 @@ def make_itanium_abi_triple(self, triple): m = re.match(r'(\w+)-(\w+)-(\w+)', triple) if not m: - self.lit_config.fatal("Could not turn '%s' into Itanium ABI triple" % triple) + self.lit_config.fatal( + "Could not turn '%s' into Itanium ABI triple" % triple) if m.group(3).lower() != 'win32': - # All non-win32 triples use the Itanium ABI. - return triple + # All non-win32 triples use the Itanium ABI. + return triple return m.group(1) + '-' + m.group(2) + '-mingw32' def make_msabi_triple(self, triple): m = re.match(r'(\w+)-(\w+)-(\w+)', triple) if not m: - self.lit_config.fatal("Could not turn '%s' into MS ABI triple" % triple) + self.lit_config.fatal( + "Could not turn '%s' into MS ABI triple" % triple) isa = m.group(1).lower() vendor = m.group(2).lower() os = m.group(3).lower() if os == 'win32': - # If the OS is win32, we're done. - return triple + # If the OS is win32, we're done. + return triple if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa): - # For x86 ISAs, adjust the OS. - return isa + '-' + vendor + '-win32' + # For x86 ISAs, adjust the OS. + return isa + '-' + vendor + '-win32' # -win32 is not supported for non-x86 targets; use a default. return 'i686-pc-win32' - def add_tool_substitutions(self, tools, search_dirs, warn_missing = True): + def add_tool_substitutions(self, tools, search_dirs=None): + if not search_dirs: + search_dirs = [self.config.llvm_tools_dir] + if lit.util.is_string(search_dirs): search_dirs = [search_dirs] + tools = [x if isinstance(x, ToolSubst) else ToolSubst(x) + for x in tools] + search_dirs = os.pathsep.join(search_dirs) + substitutions = [] + for tool in tools: - # Extract the tool name from the pattern. This relies on the tool - # name being surrounded by \b word match operators. If the - # pattern starts with "| ", include it in the string to be - # substituted. - if lit.util.is_string(tool): - tool = lit.util.make_word_regex(tool) - else: - tool = str(tool) + match = tool.resolve(self, search_dirs) - tool_match = re.match(r"^(\\)?((\| )?)\W+b([0-9A-Za-z-_\.]+)\\b\W*$", - tool) - if not tool_match: + # Either no match occurred, or there was an unresolved match that + # is ignored. + if not match: continue - tool_pipe = tool_match.group(2) - tool_name = tool_match.group(4) - tool_path = lit.util.which(tool_name, search_dirs) - if not tool_path: - if warn_missing: - # Warn, but still provide a substitution. - self.lit_config.note('Did not find ' + tool_name + ' in %s' % search_dirs) - tool_path = self.config.llvm_tools_dir + '/' + tool_name - - if tool_name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1': - tool_path += ' -verify-machineinstrs' - if tool_name == 'llvm-go': - exe = getattr(self.config, 'go_executable', None) - if exe: - tool_path += " go=" + exe - - self.config.substitutions.append((tool, tool_pipe + tool_path)) + subst_key, tool_pipe, command = match + + # An unresolved match occurred that can't be ignored. Fail without + # adding any of the previously-discovered substitutions. + if not command: + return False + + substitutions.append((subst_key, tool_pipe + command)) + + self.config.substitutions.extend(substitutions) + return True + + def use_default_substitutions(self): + tool_patterns = [ + ToolSubst('FileCheck', unresolved='fatal'), + # Handle these specially as they are strings searched for during testing. + ToolSubst(r'\| \bcount\b', command=FindTool( + 'count'), verbatim=True, unresolved='fatal'), + ToolSubst(r'\| \bnot\b', command=FindTool('not'), verbatim=True, unresolved='fatal')] + + self.config.substitutions.append(('%python', sys.executable)) + self.add_tool_substitutions( + tool_patterns, [self.config.llvm_tools_dir]) Index: llvm/utils/lit/lit/llvm/subst.py =================================================================== --- /dev/null +++ llvm/utils/lit/lit/llvm/subst.py @@ -0,0 +1,140 @@ +import os +import re + +import lit.util + +expr = re.compile(r"^(\\)?((\| )?)\W+b(\S+)\\b\W*$") +wordifier = re.compile(r"(\W*)(\b[^\b]+\b)") + + +class FindTool(object): + def __init__(self, name): + self.name = name + + def resolve(self, config, dirs): + command = lit.util.which(self.name, dirs) + if not command: + return None + + if self.name == 'llc' and os.environ.get('LLVM_ENABLE_MACHINE_VERIFIER') == '1': + command += ' -verify-machineinstrs' + elif self.name == 'llvm-go': + exe = getattr(config.config, 'go_executable', None) + if exe: + command += ' go=' + exe + return command + + +class ToolSubst(object): + """String-like class used to build regex substitution patterns for llvm + tools. + + Handles things like adding word-boundary patterns, and filtering + characters from the beginning an end of a tool name + + """ + + def __init__(self, key, command=None, pre=r'.-^/\<', post='-.', verbatim=False, + unresolved='warn', extra_args=None): + """Construct a ToolSubst. + + key: The text which is to be substituted. + + command: The command to substitute when the key is matched. By default, + this will treat `key` as a tool name and search for it. If it is + a string, it is intereprted as an exact path. If it is an instance of + FindTool, the specified tool name is searched for on disk. + + pre: If specified, the substitution will not find matches where + the character immediately preceding the word-boundary that begins + `key` is any of the characters in the string `pre`. + + post: If specified, the substitution will not find matches where + the character immediately after the word-boundary that ends `key` + is any of the characters specified in the string `post`. + + verbatim: If True, `key` is an exact regex that is passed to the + underlying substitution + + unresolved: Action to take if the tool substitution cannot be + resolved. Valid values: + 'warn' - log a warning but add the substitution anyway. + 'fatal' - Exit the test suite and log a fatal error. + 'break' - Don't add any of the substitutions from the current + group, and return a value indicating a failure. + 'ignore' - Don't add the substitution, and don't log an error + + extra_args: If specified, represents a list of arguments that will be + appended to the tool's substitution. + + explicit_path: If specified, the exact path will be used as a substitution. + Otherwise, the tool will be searched for as if by calling which(tool) + + """ + self.unresolved = unresolved + self.extra_args = extra_args + self.key = key + self.command = command if command is not None else FindTool(key) + if verbatim: + self.regex = key + return + + def not_in(chars, where=''): + if not chars: + return '' + pattern_str = '|'.join(re.escape(x) for x in chars) + return r'(?{}!({}))'.format(where, pattern_str) + + def wordify(word): + match = wordifier.match(word) + introducer = match.group(1) + word = match.group(2) + return introducer + r'\b' + word + r'\b' + + self.regex = not_in(pre, '<') + wordify(key) + not_in(post) + + def resolve(self, config, search_dirs): + # Extract the tool name from the pattern. This relies on the tool + # name being surrounded by \b word match operators. If the + # pattern starts with "| ", include it in the string to be + # substituted. + + tool_match = expr.match(self.regex) + if not tool_match: + return None + + tool_pipe = tool_match.group(2) + tool_name = tool_match.group(4) + + if isinstance(self.command, FindTool): + command_str = self.command.resolve(config, search_dirs) + else: + command_str = str(self.command) + + if command_str: + if self.extra_args: + command_str = ' '.join([command_str] + self.extra_args) + else: + if self.unresolved == 'warn': + # Warn, but still provide a substitution. + config.lit_config.note( + 'Did not find ' + tool_name + ' in %s' % search_dirs) + command_str = os.path.join( + config.config.llvm_tools_dir, tool_name) + elif self.unresolved == 'fatal': + # The function won't even return in this case, this leads to + # sys.exit + config.lit_config.fatal( + 'Did not find ' + tool_name + ' in %s' % search_dirs) + elif self.unresolved == 'break': + # By returning a valid result with an empty command, the + # caller treats this as a failure. + pass + elif self.unresolved == 'ignore': + # By returning None, the caller just assumes there was no + # match in the first place. + return None + else: + raise 'Unexpected value for ToolSubst.unresolved' + + return (self.regex, tool_pipe, command_str) Index: llvm/utils/lit/lit/util.py =================================================================== --- llvm/utils/lit/lit/util.py +++ llvm/utils/lit/lit/util.py @@ -9,12 +9,14 @@ import sys import threading + def norm_path(path): path = os.path.realpath(path) path = os.path.normpath(path) path = os.path.normcase(path) return path + def is_string(value): try: # Python 2 and Python 3 are different here. @@ -22,6 +24,7 @@ except NameError: return isinstance(value, str) + def pythonize_bool(value): if value is None: return False @@ -36,14 +39,17 @@ return False raise ValueError('"{}" is not a valid boolean'.format(value)) + def make_word_regex(word): return r'\b' + word + r'\b' + def to_bytes(s): """Return the parameter as type 'bytes', possibly encoding it. - In Python2, the 'bytes' type is the same as 'str'. In Python3, they are - distinct. + In Python2, the 'bytes' type is the same as 'str'. In Python3, they + are distinct. + """ if isinstance(s, bytes): # In Python2, this branch is taken for both 'str' and 'bytes'. @@ -54,12 +60,14 @@ # Encode to UTF-8 to get 'bytes' data. return s.encode('utf-8') + def to_string(b): """Return the parameter as type 'str', possibly encoding it. In Python2, the 'str' type is the same as 'bytes'. In Python3, the 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is distinct. + """ if isinstance(b, str): # In Python2, this branch is taken for types 'str' and 'bytes'. @@ -91,28 +99,32 @@ except AttributeError: raise TypeError('not sure how to convert %s to %s' % (type(b), str)) + def detectCPUs(): - """ - Detects the number of CPUs on a system. Cribbed from pp. + """Detects the number of CPUs on a system. + + Cribbed from pp. + """ # Linux, Unix and MacOS: - if hasattr(os, "sysconf"): - if "SC_NPROCESSORS_ONLN" in os.sysconf_names: + if hasattr(os, 'sysconf'): + if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: # Linux & Unix: - ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + ncpus = os.sysconf('SC_NPROCESSORS_ONLN') if isinstance(ncpus, int) and ncpus > 0: return ncpus - else: # OSX: + else: # OSX: return int(subprocess.check_output(['sysctl', '-n', 'hw.ncpu'], stderr=subprocess.STDOUT)) # Windows: - if "NUMBER_OF_PROCESSORS" in os.environ: - ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if 'NUMBER_OF_PROCESSORS' in os.environ: + ncpus = int(os.environ['NUMBER_OF_PROCESSORS']) if ncpus > 0: # With more than 32 processes, process creation often fails with # "Too many open files". FIXME: Check if there's a better fix. return min(ncpus, 32) - return 1 # Default + return 1 # Default + def mkdir_p(path): """mkdir_p(path) - Make the "path" directory, if it does not exist; this @@ -132,6 +144,7 @@ if e.errno != errno.EEXIST: raise + def listdir_files(dirname, suffixes=None, exclude_filenames=None): """Yields files in a directory. @@ -158,6 +171,7 @@ Yields: Filenames as returned by os.listdir (generally, str). + """ if exclude_filenames is None: exclude_filenames = set() @@ -167,20 +181,21 @@ if (os.path.isdir(os.path.join(dirname, filename)) or filename.startswith('.') or filename in exclude_filenames or - not any(filename.endswith(sfx) for sfx in suffixes)): + not any(filename.endswith(sfx) for sfx in suffixes)): continue yield filename -def which(command, paths = None): + +def which(command, paths=None): """which(command, [paths]) - Look up the given command in the paths string (or the PATH environment variable, if unspecified).""" if paths is None: - paths = os.environ.get('PATH','') + paths = os.environ.get('PATH', '') # Check for absolute match first. if os.path.isfile(command): - return command + return os.path.normpath(command) # Would be nice if Python had a lib function for this. if not paths: @@ -198,26 +213,29 @@ for ext in pathext: p = os.path.join(path, command + ext) if os.path.exists(p) and not os.path.isdir(p): - return p + return os.path.normpath(p) return None + def checkToolsPath(dir, tools): for tool in tools: if not os.path.exists(os.path.join(dir, tool)): return False return True + def whichTools(tools, paths): for path in paths.split(os.pathsep): if checkToolsPath(path, tools): return path return None -def printHistogram(items, title = 'Items'): - items.sort(key = lambda item: item[1]) - maxValue = max([v for _,v in items]) +def printHistogram(items, title='Items'): + items.sort(key=lambda item: item[1]) + + maxValue = max([v for _, v in items]) # Select first "nice" bar height that produces more than 10 bars. power = int(math.ceil(math.log(maxValue, 10))) @@ -230,33 +248,34 @@ power -= 1 histo = [set() for i in range(N)] - for name,v in items: - bin = min(int(N * v/maxValue), N-1) + for name, v in items: + bin = min(int(N * v / maxValue), N - 1) histo[bin].add(name) barW = 40 hr = '-' * (barW + 34) print('\nSlowest %s:' % title) print(hr) - for name,value in items[-20:]: + for name, value in items[-20:]: print('%.2fs: %s' % (value, name)) print('\n%s Times:' % title) print(hr) pDigits = int(math.ceil(math.log(maxValue, 10))) - pfDigits = max(0, 3-pDigits) + pfDigits = max(0, 3 - pDigits) if pfDigits: pDigits += pfDigits + 1 cDigits = int(math.ceil(math.log(len(items), 10))) - print("[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3), + print('[%s] :: [%s] :: [%s]' % ('Range'.center((pDigits + 1) * 2 + 3), 'Percentage'.center(barW), - 'Count'.center(cDigits*2 + 1))) + 'Count'.center(cDigits * 2 + 1))) print(hr) - for i,row in enumerate(histo): + for i, row in enumerate(histo): pct = float(len(row)) / len(items) w = int(barW * pct) - print("[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" % ( - pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH, - '*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items))) + print('[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]' % ( + pDigits, pfDigits, i * barH, pDigits, pfDigits, (i + 1) * barH, + '*' * w, ' ' * (barW - w), cDigits, len(row), cDigits, len(items))) + class ExecuteCommandTimeoutException(Exception): def __init__(self, msg, out, err, exitCode): @@ -269,27 +288,30 @@ self.err = err self.exitCode = exitCode + # Close extra file handles on UNIX (on Windows this cannot be done while # also redirecting input). kUseCloseFDs = not (platform.system() == 'Windows') + + def executeCommand(command, cwd=None, env=None, input=None, timeout=0): - """ - Execute command ``command`` (list of arguments or string) - with - * working directory ``cwd`` (str), use None to use the current - working directory - * environment ``env`` (dict), use None for none - * Input to the command ``input`` (str), use string to pass - no input. - * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. - - Returns a tuple (out, err, exitCode) where - * ``out`` (str) is the standard output of running the command - * ``err`` (str) is the standard error of running the command - * ``exitCode`` (int) is the exitCode of running the command - - If the timeout is hit an ``ExecuteCommandTimeoutException`` - is raised. + """Execute command ``command`` (list of arguments or string) with. + + * working directory ``cwd`` (str), use None to use the current + working directory + * environment ``env`` (dict), use None for none + * Input to the command ``input`` (str), use string to pass + no input. + * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. + + Returns a tuple (out, err, exitCode) where + * ``out`` (str) is the standard output of running the command + * ``err`` (str) is the standard error of running the command + * ``exitCode`` (int) is the exitCode of running the command + + If the timeout is hit an ``ExecuteCommandTimeoutException`` + is raised. + """ if input is not None: input = to_bytes(input) @@ -315,7 +337,7 @@ timerObject = threading.Timer(timeout, killProcess) timerObject.start() - out,err = p.communicate(input=input) + out, err = p.communicate(input=input) exitCode = p.wait() finally: if timerObject != None: @@ -331,7 +353,7 @@ out=out, err=err, exitCode=exitCode - ) + ) # Detect Ctrl-C in subprocess. if exitCode == -signal.SIGINT: @@ -339,6 +361,7 @@ return out, err, exitCode + def usePlatformSdkOnDarwin(config, lit_config): # On Darwin, support relocatable SDKs by providing Clang with a # default system root path. @@ -356,6 +379,7 @@ lit_config.note('using SDKROOT: %r' % sdk_path) config.environment['SDKROOT'] = sdk_path + def findPlatformSdkVersionOnMacOS(config, lit_config): if 'darwin' in config.target_triple: try: @@ -370,15 +394,15 @@ return out return None + def killProcessAndChildren(pid): - """ - This function kills a process with ``pid`` and all its - running children (recursively). It is currently implemented - using the psutil module which provides a simple platform - neutral implementation. + """This function kills a process with ``pid`` and all its running children + (recursively). It is currently implemented using the psutil module which + provides a simple platform neutral implementation. + + TODO: Reimplement this without using psutil so we can remove + our dependency on it. - TODO: Reimplement this without using psutil so we can - remove our dependency on it. """ import psutil try: