Index: CMakeLists.txt =================================================================== --- CMakeLists.txt +++ CMakeLists.txt @@ -433,6 +433,20 @@ set(LLVM_ADD_NATIVE_VISUALIZERS_TO_SOLUTION FALSE CACHE INTERNAL "For Visual Studio 2013, manually copy natvis files to Documents\\Visual Studio 2013\\Visualizers" FORCE) endif() +if (LLVM_BUILD_INSTRUMENTED_COVERAGE) + if(NOT HOST_LLVM_PROFDATA) + find_program(HOST_LLVM_PROFDATA llvm-profdata) + endif() + + if(NOT HOST_LLVM_PROFDATA) + message(FATAL_ERROR "Must set HOST_LLVM_PROFDATA to point to llvm-profdata to use for merging PGO data") + endif() + + if(NOT LLVM_PROFILE_DATA_DIR) + message(FATAL_ERROR "Must set LLVM_PROFILE_DATA_DIR to point to a directory") + endif() +endif() + # All options referred to from HandleLLVMOptions have to be specified # BEFORE this include, otherwise options will not be correctly set on # first cmake run Index: cmake/modules/HandleLLVMOptions.cmake =================================================================== --- cmake/modules/HandleLLVMOptions.cmake +++ cmake/modules/HandleLLVMOptions.cmake @@ -605,6 +605,14 @@ CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) +option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation (experimental)" Off) +mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE) +append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate -fcoverage-mapping" + CMAKE_CXX_FLAGS + CMAKE_C_FLAGS + CMAKE_EXE_LINKER_FLAGS + CMAKE_SHARED_LINKER_FLAGS) + set(LLVM_ENABLE_LTO OFF CACHE STRING "Build LLVM with LTO. May be specified as Thin or Full to use a particular kind of LTO") string(TOUPPER "${LLVM_ENABLE_LTO}" uppercase_LLVM_ENABLE_LTO) if(uppercase_LLVM_ENABLE_LTO STREQUAL "THIN") Index: test/Unit/lit.site.cfg.in =================================================================== --- test/Unit/lit.site.cfg.in +++ test/Unit/lit.site.cfg.in @@ -8,6 +8,9 @@ config.llvm_build_mode = "@LLVM_BUILD_MODE@" config.enable_shared = @ENABLE_SHARED@ config.shlibdir = "@SHLIBDIR@" +config.host_llvm_profdata = "@HOST_LLVM_PROFDATA@" +config.collect_profiles = "@LLVM_BUILD_INSTRUMENTED_COVERAGE@".lower() == 'on' +config.profile_data_dir = "@LLVM_PROFILE_DATA_DIR@" # Support substitution of the tools_dir and build_mode with user parameters. # This is used when we can't determine the tool dir at configuration time. Index: test/lit.site.cfg.in =================================================================== --- test/lit.site.cfg.in +++ test/lit.site.cfg.in @@ -30,6 +30,7 @@ config.host_arch = "@HOST_ARCH@" config.host_cc = "@HOST_CC@" config.host_cxx = "@HOST_CXX@" +config.host_llvm_profdata = "@HOST_LLVM_PROFDATA@" config.host_ldflags = "@HOST_LDFLAGS@" config.llvm_use_intel_jitevents = "@LLVM_USE_INTEL_JITEVENTS@" config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@" @@ -38,6 +39,8 @@ config.have_dia_sdk = @HAVE_DIA_SDK@ config.enable_ffi = "@LLVM_ENABLE_FFI@" config.test_examples = "@ENABLE_EXAMPLES@" +config.collect_profiles = "@LLVM_BUILD_INSTRUMENTED_COVERAGE@".lower() == 'on' +config.profile_data_dir = "@LLVM_PROFILE_DATA_DIR@" # Support substitution of the tools_dir with user parameters. This is # used when we can't determine the tool dir at configuration time. Index: utils/lit/lit/LitConfig.py =================================================================== --- utils/lit/lit/LitConfig.py +++ utils/lit/lit/LitConfig.py @@ -59,6 +59,10 @@ self.valgrindArgs.append('--leak-check=no') self.valgrindArgs.extend(self.valgrindUserArgs) + self.collect_profiles = False + self.host_llvm_profdata = '' + self.profile_data_dir = '' + self.maxIndividualTestTime = maxIndividualTestTime @property @@ -126,6 +130,15 @@ return dir + def isProfileCollectionEnabled(self): + return self.collect_profiles + + def getProfileDataDir(self): + return self.profile_data_dir + + def getHostLLVMProfdata(self): + return self.host_llvm_profdata + def _write_message(self, kind, message): # Get the file/line where this message was generated. f = inspect.currentframe() Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -120,7 +120,7 @@ self.exitCode = exitCode self.timeoutReached = timeoutReached -def executeShCmd(cmd, shenv, results, timeout=0): +def executeShCmd(cmd, shenv, results, litConfig, timeout=0): """ Wrapper around _executeShCmd that handles timeout @@ -130,7 +130,7 @@ timeoutHelper = TimeoutHelper(timeout) if timeout > 0: timeoutHelper.startTimer() - finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper) + finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper, litConfig) timeoutHelper.cancel() timeoutInfo = None if timeoutHelper.timeoutReached(): @@ -138,7 +138,7 @@ return (finalExitCode, timeoutInfo) -def _executeShCmd(cmd, shenv, results, timeoutHelper): +def _executeShCmd(cmd, shenv, results, timeoutHelper, litConfig): if timeoutHelper.timeoutReached(): # Prevent further recursion if the timeout has been hit # as we should try avoid launching more processes. @@ -146,25 +146,25 @@ if isinstance(cmd, ShUtil.Seq): if cmd.op == ';': - res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) - return _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) + res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper, litConfig) + return _executeShCmd(cmd.rhs, shenv, results, timeoutHelper, litConfig) if cmd.op == '&': raise InternalShellError(cmd,"unsupported shell operator: '&'") if cmd.op == '||': - res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) + res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper, litConfig) if res != 0: - res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) + res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper, litConfig) return res if cmd.op == '&&': - res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper) + res = _executeShCmd(cmd.lhs, shenv, results, timeoutHelper, litConfig) if res is None: return res if res == 0: - res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper) + res = _executeShCmd(cmd.rhs, shenv, results, timeoutHelper, litConfig) return res raise ValueError('Unknown shell command: %r' % cmd.op) @@ -313,28 +313,28 @@ args[i] = f.name try: - procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd, - executable = executable, - stdin = stdin, - stdout = stdout, - stderr = stderr, - env = cmd_shenv.env, - close_fds = kUseCloseFDs)) + IP = InstrumentedProcess.create(args, litConfig, cmd_shenv.cwd, + executable, stdin, stdout, stderr, + cmd_shenv.env, kUseCloseFDs) + procs.append(IP) + # Let the helper know about this process - timeoutHelper.addProcess(procs[-1]) + timeoutHelper.addProcess(IP.getProcess()) except OSError as e: raise InternalShellError(j, 'Could not create process ({}) due to {}'.format(executable, e)) + proc_last = procs[-1].getProcess() + # Immediately close stdin for any process taking stdin from us. if stdin == subprocess.PIPE: - procs[-1].stdin.close() - procs[-1].stdin = None + proc_last.stdin.close() + proc_last.stdin = None # Update the current stdin source. if stdout == subprocess.PIPE: - input = procs[-1].stdout + input = proc_last.stdout elif stderrIsStdout: - input = procs[-1].stderr + input = proc_last.stderr else: input = subprocess.PIPE @@ -347,15 +347,16 @@ # FIXME: There is probably still deadlock potential here. Yawn. procData = [None] * len(procs) - procData[-1] = procs[-1].communicate() + procData[-1] = proc_last.communicate() for i in range(len(procs) - 1): - if procs[i].stdout is not None: - out = procs[i].stdout.read() + proc_i = procs[i].getProcess() + if proc_i.stdout is not None: + out = proc_i.stdout.read() else: out = '' - if procs[i].stderr is not None: - err = procs[i].stderr.read() + if proc_i.stderr is not None: + err = proc_i.stderr.read() else: err = '' procData[i] = (out,err) @@ -372,11 +373,14 @@ exitCode = None for i,(out,err) in enumerate(procData): - res = procs[i].wait() + IP = procs[i] + res = IP.getProcess().wait() # Detect Ctrl-C in subprocess. if res == -signal.SIGINT: raise KeyboardInterrupt + IP.cleanup() + # Ensure the resulting output is always of string type. try: if out is None: @@ -435,7 +439,7 @@ timeoutInfo = None try: shenv = ShellEnvironment(cwd, test.config.environment) - exitCode, timeoutInfo = executeShCmd(cmd, shenv, results, timeout=litConfig.maxIndividualTestTime) + exitCode, timeoutInfo = executeShCmd(cmd, shenv, results, litConfig, timeout=litConfig.maxIndividualTestTime) except InternalShellError: e = sys.exc_info()[1] exitCode = 127 @@ -506,7 +510,7 @@ command = litConfig.valgrindArgs + command try: - out, err, exitCode = lit.util.executeCommand(command, cwd=cwd, + out, err, exitCode = lit.util.executeCommand(command, litConfig, cwd=cwd, env=test.config.environment, timeout=litConfig.maxIndividualTestTime) return (out, err, exitCode, None) Index: utils/lit/lit/TestingConfig.py =================================================================== --- utils/lit/lit/TestingConfig.py +++ utils/lit/lit/TestingConfig.py @@ -155,6 +155,12 @@ # files. Should we distinguish them? self.test_source_root = str(self.test_source_root) self.excludes = set(self.excludes) + if self.collect_profiles: + litConfig.collect_profiles = True + if self.host_llvm_profdata: + litConfig.host_llvm_profdata = self.host_llvm_profdata + if self.profile_data_dir: + litConfig.profile_data_dir = self.profile_data_dir @property def root(self): Index: utils/lit/lit/formats/base.py =================================================================== --- utils/lit/lit/formats/base.py +++ utils/lit/lit/formats/base.py @@ -101,7 +101,7 @@ else: cmd.append(test.getSourcePath()) - out, err, exitCode = lit.util.executeCommand(cmd) + out, err, exitCode = lit.util.executeCommand(cmd, litConfig) diags = out + err if not exitCode and not diags.strip(): Index: utils/lit/lit/formats/googletest.py =================================================================== --- utils/lit/lit/formats/googletest.py +++ utils/lit/lit/formats/googletest.py @@ -117,7 +117,7 @@ try: out, err, exitCode = lit.util.executeCommand( - cmd, env=test.config.environment, + cmd, litConfig, env=test.config.environment, timeout=litConfig.maxIndividualTestTime) except lit.util.ExecuteCommandTimeoutException: return (lit.Test.TIMEOUT, Index: utils/lit/lit/util.py =================================================================== --- utils/lit/lit/util.py +++ utils/lit/lit/util.py @@ -7,6 +7,8 @@ import subprocess import sys import threading +import hashlib +import glob def to_bytes(str): # Encode to UTF-8 to get binary data. @@ -114,6 +116,74 @@ return path return None +class InstrumentedProcess(object): + '''Wrapper object around a subprocess.Popen instance. + + If profiling instrumentation is enabled (e.g for PGO or Code Coverage), + this object can be used to index any raw profiles the process creates + (and then clean up). + ''' + def __init__(self, process, base_profile_path, litConfig): + '''Warning: This initializer shouldn't be used directly. See + InstrumentedProcess.create(...). + ''' + self.process = process + self.base_profile_path = base_profile_path + self.litConfig = litConfig + + @staticmethod + def _getBaseProfilePath(command, litConfig): + '''Find a reasonable place to put profiles based on the command.''' + prfhash = hashlib.sha224(''.join(command)).hexdigest() + return os.path.join(litConfig.getProfileDataDir(), prfhash) + + @staticmethod + def create(command, litConfig, cwd=None, executable=None, stdin=None, + stdout=None, stderr=None, env=None, close_fds=False): + '''Create an InstrumentedProcess instance. Apart from litConfig, the + arguments are identical to the ones passed to subprocess.Popen(...). + ''' + if not litConfig.isProfileCollectionEnabled(): + modified_env = env + base_profile_path = '' + else: + base_profile_path = InstrumentedProcess._getBaseProfilePath( + command, litConfig) + raw_profile_path = base_profile_path + ':%p.profraw' + modified_env = env or {} + modified_env['LLVM_PROFILE_FILE'] = raw_profile_path + process = subprocess.Popen(command, cwd=cwd, executable=executable, + stdin=stdin, stdout=stdout, stderr=stderr, + env=modified_env, close_fds=close_fds) + return InstrumentedProcess(process, base_profile_path, litConfig) + + def getProcess(self): + '''Get the underyling subprocess.Popen instance.''' + return self.process + + def cleanup(self): + '''Index and delete all raw profiles generated by this process.''' + if not self.litConfig.isProfileCollectionEnabled(): + return + + raw_profiles = glob.glob(self.base_profile_path + '*') + if not len(raw_profiles): + return + + # Merge all of the generated raw profiles together. + merge_tool = [self.litConfig.getHostLLVMProfdata(), 'merge', '-sparse'] + indexed_profile_path = self.base_profile_path + '.profdata' + try: + cmd = ' '.join(merge_tool + raw_profiles + ['-o', + indexed_profile_path]) + out = subprocess.check_output(cmd, shell=True) + except OSError as OE: + raise OSError(cmd) + + # Clean up after ourselves. + for raw_profile in raw_profiles: + os.remove(raw_profile) + def printHistogram(items, title = 'Items'): items.sort(key = lambda item: item[1]) @@ -172,7 +242,7 @@ # Close extra file handles on UNIX (on Windows this cannot be done while # also redirecting input). kUseCloseFDs = not (platform.system() == 'Windows') -def executeCommand(command, cwd=None, env=None, input=None, timeout=0): +def executeCommand(command, litConfig, cwd=None, env=None, input=None, timeout=0): """ Execute command ``command`` (list of arguments or string) with @@ -191,11 +261,13 @@ If the timeout is hit an ``ExecuteCommandTimeoutException`` is raised. """ - p = subprocess.Popen(command, cwd=cwd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, close_fds=kUseCloseFDs) + IP = InstrumentedProcess.create(command, litConfig, cwd=cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, + close_fds=kUseCloseFDs) + p = IP.getProcess() + timerObject = None # FIXME: Because of the way nested function scopes work in Python 2.x we # need to use a reference to a mutable object rather than a plain @@ -215,6 +287,7 @@ out,err = p.communicate(input=input) exitCode = p.wait() + IP.cleanup() finally: if timerObject != None: timerObject.cancel() Index: utils/prepare-code-coverage-artifact.sh =================================================================== --- /dev/null +++ utils/prepare-code-coverage-artifact.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +if [ "$#" -ne 4 ]; then + echo "Usage:" + echo "$0 " + exit 1 +fi + +HOST_LLVM_PROFDATA=$1 +HOST_LLVM_COV=$2 +LLVM_PROFILE_DATA_DIR=$3 +LLVM_BIN_DIR=$4 + +echo ":: llvm-profdata = " $HOST_LLVM_PROFDATA +echo ":: llvm-cov = " $HOST_LLVM_COV + +echo ":: Merging indexed profiles in $LLVM_PROFILE_DATA_DIR" +find $LLVM_PROFILE_DATA_DIR -name \*.profdata > $LLVM_PROFILE_DATA_DIR/profiles.list +$HOST_LLVM_PROFDATA merge -sparse -f $LLVM_PROFILE_DATA_DIR/profiles.list -o $LLVM_PROFILE_DATA_DIR/Coverage.aggregate + +echo ":: Removing indexed profiles in $LLVM_PROFILE_DATA_DIR" +find $LLVM_PROFILE_DATA_DIR -name \*.profdata -exec rm {} \; +mv $LLVM_PROFILE_DATA_DIR/Coverage.aggregate $LLVM_PROFILE_DATA_DIR/Coverage.profdata + +echo ":: Copying covmappings from $LLVM_BIN_DIR into $LLVM_PROFILE_DATA_DIR" +for prog in $(find $LLVM_BIN_DIR -type f | grep -v llvm-lit); do + $HOST_LLVM_COV convert-for-testing $prog -o $LLVM_PROFILE_DATA_DIR/$(basename $prog).covmapping +done