Index: CMakeLists.txt =================================================================== --- CMakeLists.txt +++ CMakeLists.txt @@ -71,3 +71,6 @@ add_subdirectory(SingleSource) add_subdirectory(MultiSource) + +# Produce lit.site.cfg +configure_file("${CMAKE_SOURCE_DIR}/lit.site.cfg.in" "${CMAKE_BINARY_DIR}/lit.site.cfg") Index: cmake/lit-test-template.in =================================================================== --- cmake/lit-test-template.in +++ cmake/lit-test-template.in @@ -1,3 +1,3 @@ -; RUN: ${RUNUNDER} ${CMAKE_SOURCE_DIR}/RunSafely.sh -t ${TIMEIT} 7200 ${STDIN_FILENAME} %t ${CMAKE_CURRENT_BINARY_DIR}/${exename} ${RUN_OPTIONS} -; RUN: ${PROGRAM_OUTPUT_FILTER} %t -; RUN: ${DIFFPROG} %t ${REFERENCE_OUTPUT} +; RUN: ${CMAKE_CURRENT_BINARY_DIR}/${exename} ${RUN_OPTIONS} < ${STDIN_FILENAME} +; VERIFY: ${PROGRAM_OUTPUT_FILTER} %t +; VERIFY: ${DIFFPROG} %t ${REFERENCE_OUTPUT} Index: lit.cfg =================================================================== --- lit.cfg +++ lit.cfg @@ -1,6 +1,13 @@ import lit.formats +import lit.util import lit import os, glob, re +import shlex +import pipes +from lit.formats import FileBasedTest +from lit.TestRunner import executeScriptInternal, kIsWindows +from lit import Test +from lit.util import to_bytes, to_string def getUserTimeFromTimeOutput(f): with open(f) as fd: @@ -11,36 +18,302 @@ m = re.match(r'user\s+([0-9.]+)', l[0]) return float(m.group(1)) -class TestSuiteTest(lit.formats.ShTest): +def getTmpDirBase(test): + execpath = test.getExecPath() + execdir,execbase = os.path.split(execpath) + tmpDir = os.path.join(execdir, 'Output') + tmpBase = os.path.join(tmpDir, execbase) + return tmpDir, tmpBase + +def parseIntegratedTestScriptCommands(source_path): + """ + parseIntegratedTestScriptCommands(source_path) -> commands + + Parse the commands in an integrated test script file into a list of + (line_number, command_type, line). + """ + + # This code is carefully written to be dual compatible with Python 2.5+ and + # Python 3 without requiring input files to always have valid codings. The + # trick we use is to open the file in binary mode and use the regular + # expression library to find the commands, with it scanning strings in + # Python2 and bytes in Python3. + # + # Once we find a match, we do require each script line to be decodable to + # UTF-8, so we convert the outputs to UTF-8 before returning. This way the + # remaining code can work with "strings" agnostic of the executing Python + # version. + + keywords = ['RUN:', 'VERIFY:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.'] + keywords_re = re.compile( + to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) + + f = open(source_path, 'rb') + try: + # Read the entire file contents. + data = f.read() + + # Ensure the data ends with a newline. + if not data.endswith(to_bytes('\n')): + data = data + to_bytes('\n') + + # Iterate over the matches. + line_number = 1 + last_match_position = 0 + for match in keywords_re.finditer(data): + # Compute the updated line number by counting the intervening + # newlines. + match_position = match.start() + line_number += data.count(to_bytes('\n'), last_match_position, + match_position) + last_match_position = match_position + + # Convert the keyword and line to UTF-8 strings and yield the + # command. Note that we take care to return regular strings in + # Python 2, to avoid other code having to differentiate between the + # str and unicode types. + keyword,ln = match.groups() + yield (line_number, to_string(keyword[:-1].decode('utf-8')), + to_string(ln.decode('utf-8'))) + finally: + f.close() + +def parseBenchmarkScript(test, extra_substitutions=[]): + """Scan a llvm-testsuite like benchmark .test script.""" + sourcepath = test.getSourcePath() + sourcedir = os.path.dirname(sourcepath) + execpath = test.getExecPath() + execdir,execbase = os.path.split(execpath) + tmpDir, tmpBase = getTmpDirBase(test) + + # We use #_MARKER_# to hide %% while we do the other substitutions. + substitutions = list(extra_substitutions) + substitutions.extend([('%%', '#_MARKER_#')]) + substitutions.extend(test.config.substitutions) + substitutions.extend([('%s', sourcepath), + ('%S', sourcedir), + ('%p', sourcedir), + ('%{pathsep}', os.pathsep), + ('%t', tmpBase + '.tmp'), + ('%T', tmpDir), + ('#_MARKER_#', '%')]) + + # "%/[STpst]" should be normalized. + substitutions.extend([ + ('%/s', sourcepath.replace('\\', '/')), + ('%/S', sourcedir.replace('\\', '/')), + ('%/p', sourcedir.replace('\\', '/')), + ('%/t', tmpBase.replace('\\', '/') + '.tmp'), + ('%/T', tmpDir.replace('\\', '/')), + ]) + + def parseShellCommand(script, ln): + # Trim trailing whitespace. + ln = ln.rstrip() + + # Substitute line number expressions + ln = re.sub('%\(line\)', str(line_number), ln) + def replace_line_number(match): + if match.group(1) == '+': + return str(line_number + int(match.group(2))) + if match.group(1) == '-': + return str(line_number - int(match.group(2))) + ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln) + + # Collapse lines with trailing '\\'. + if script and script[-1][-1] == '\\': + script[-1] = script[-1][:-1] + ln + else: + script.append(ln) + + # Collect the test lines from the script. + runscript = [] + verifyscript = [] + requires = [] + unsupported = [] + for line_number, command_type, ln in \ + parseIntegratedTestScriptCommands(sourcepath): + if command_type == 'RUN': + parseShellCommand(runscript, ln) + elif command_type == 'VERIFY': + parseShellCommand(verifyscript, ln) + elif command_type == 'XFAIL': + test.xfails.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'REQUIRES': + requires.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'UNSUPPORTED': + unsupported.extend([s.strip() for s in ln.split(',')]) + elif command_type == 'END': + # END commands are only honored if the rest of the line is empty. + if not ln.strip(): + break + else: + raise ValueError("unknown script command type: %r" % ( + command_type,)) + + # Apply substitutions to the script. Allow full regular + # expression syntax. Replace each matching occurrence of regular + # expression pattern a with substitution b in line ln. + def processLine(ln): + # Apply substitutions + for a,b in substitutions: + if kIsWindows: + b = b.replace("\\","\\\\") + ln = re.sub(a, b, ln) + + # Strip the trailing newline and any extra whitespace. + return ln.strip() + runscript = [processLine(ln) for ln in runscript] + verifyscript = [processLine(ln) for ln in verifyscript] + + # Verify the script contains a run line. + if runscript == []: + return lit.Test.Result(Test.UNRESOLVED, "Test has no RUN: line!") + + # Check for unterminated run lines. + for script in runscript, verifyscript: + if script and script[-1][-1] == '\\': + return lit.Test.Result(Test.UNRESOLVED, + "Test has unterminated RUN/VERIFY lines (with '\\')") + + # Check that we have the required features: + missing_required_features = [f for f in requires + if f not in test.config.available_features] + if missing_required_features: + msg = ', '.join(missing_required_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires the following features: %s" % msg) + unsupported_features = [f for f in unsupported + if f in test.config.available_features] + if unsupported_features: + msg = ', '.join(unsupported_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test is unsupported with the following features: %s" % msg) + + unsupported_targets = [f for f in unsupported + if f in test.suite.config.target_triple] + if unsupported_targets: + return lit.Test.Result(Test.UNSUPPORTED, + "Test is unsupported with the following triple: %s" % ( + test.suite.config.target_triple,)) + + if test.config.limit_to_features: + # Check that we have one of the limit_to_features features in requires. + limit_to_features_tests = [f for f in test.config.limit_to_features + if f in requires] + if not limit_to_features_tests: + msg = ', '.join(test.config.limit_to_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test requires one of the limit_to_features features %s" % msg) + + return runscript,verifyscript,tmpBase,execdir + +def collectTimes(test, result): + basepath = os.path.dirname(test.getFilePath()) + if not result.code.isFailure: + # Collect the timing information. + timeglob = os.path.join(basepath, 'Output', '*.time') + times = glob.glob(timeglob) + assert len(times) == 1 + time = getUserTimeFromTimeOutput(times[0]) + + result.addMetric('exec_time', lit.Test.toMetricValue(time)) + + # For completeness, attempt to find compile time information too. + compile_time = 0.0 + for path, subdirs, files in os.walk(basepath): + for file in files: + if file.endswith('.o.time'): + compile_time += getUserTimeFromTimeOutput(os.path.join(path, file)) + result.addMetric('compile_time', lit.Test.toMetricValue(compile_time)) + +def runScript(test, litConfig, script, tmpBase, execdir, useExternalSh = False): + # Create the output directory if it does not already exist. + lit.util.mkdir_p(os.path.dirname(tmpBase)) + + if useExternalSh: + res = executeScript(test, litConfig, tmpBase, script, execdir) + else: + res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) + if isinstance(res, lit.Test.Result): + return res + + out,err,exitCode = res + # Form the output log. + output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % ( + '\n'.join(script), exitCode) + + # Append the outputs, if present. + if out: + output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) + if err: + output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) + + if exitCode == 0: + status = Test.PASS + else: + status = Test.FAIL + + result = lit.Test.Result(status, output) + if exitCode == 0: + collectTimes(test, result) + return result + +class TestSuiteTest(FileBasedTest): def __init__(self): - lit.formats.ShTest.__init__(self, False) + super(TestSuiteTest, self).__init__() def execute(self, test, litConfig): - result = lit.formats.ShTest.execute(self, test, litConfig) - basepath = os.path.dirname(test.getFilePath()) - - if not result.code.isFailure: - # Collect the timing information. - timeglob = os.path.join(basepath, 'Output', '*.time') - times = glob.glob(timeglob) - assert len(times) == 1 - time = getUserTimeFromTimeOutput(times[0]) - - result.addMetric('exec_time', lit.Test.toMetricValue(time)) - - # For completeness, attempt to find compile time information too. - compile_time = 0.0 - for path, subdirs, files in os.walk(basepath): - for file in files: - if file.endswith('.o.time'): - compile_time += getUserTimeFromTimeOutput(os.path.join(path, file)) - result.addMetric('compile_time', lit.Test.toMetricValue(compile_time)) - - return result + if test.config.unsupported: + return (Test.UNSUPPORTED, 'Test is unsupported') + + # Parse benchmark script + res = parseBenchmarkScript(test) + if isinstance(res, lit.Test.Result): + return res + if litConfig.noExecute: + return lit.Test.Result(Test.PASS) + runscript, verifyscript, tmpBase, execdir = res + + # Prepend runscript with RunSafely and timeit stuff + def prependRunSafely(line): + # Search for "< INPUTFILE" in the line and use that for stdin + stdin = "/dev/null" + commandline = shlex.split(line) + for i in range(len(commandline)): + if commandline[i] == "<" and i+1 < len(commandline): + stdin = commandline[i+1] + del commandline[i+1] + del commandline[i] + break + timeit = config.test_source_root + "/tools/timeit" + output = tmpBase + ".out" + timeout = "7200" + runsafely_prefix = [config.test_suite_root + "/RunSafely.sh", + "--show-errors", "-t", timeit, timeout, stdin, output] + + # TODO: Need an alternative to inofficial pipes.quote API. + line = " ".join(map(pipes.quote, runsafely_prefix + commandline)) + return line + runscript = map(prependRunSafely, runscript) + + # Run RUN: part of the script n times + n_runs = 1 + for n in range(n_runs): + runresult = runScript(test, litConfig, runscript, tmpBase, execdir) + if runresult.code == Test.FAIL: + return runresult + # TODO: aggregate results for multiple runs + + # Run verification of results + verifyresult = runScript(test, litConfig, verifyscript, tmpBase, execdir) + if verifyresult.code == Test.FAIL: + return verifyresult + + return runresult config.name = 'test-suite' config.test_format = TestSuiteTest() config.suffixes = ['.test'] -config.test_source_root = os.path.dirname(__file__) config.excludes = ['ABI-Testsuite'] Index: lit.site.cfg.in =================================================================== --- /dev/null +++ lit.site.cfg.in @@ -0,0 +1,6 @@ +import sys + +config.test_source_root = "@CMAKE_BINARY_DIR@" +config.test_suite_root = "@CMAKE_SOURCE_DIR@" + +lit_config.load_config(config, "@CMAKE_SOURCE_DIR@/lit.cfg")