Index: CMakeLists.txt =================================================================== --- CMakeLists.txt +++ CMakeLists.txt @@ -258,6 +258,17 @@ # Produce lit.site.cfg configure_file("${PROJECT_SOURCE_DIR}/lit.site.cfg.in" "${CMAKE_BINARY_DIR}/lit.site.cfg") +# Produce lnt run info (mostly about the used compiler) +option(TEST_SUITE_CREATE_RUNINFO + "Create compiler/run information suitable for LNT" On) +if(TEST_SUITE_CREATE_RUNINFO) + message(STATUS "Creating compilerinfo.json") + execute_process( + COMMAND ${PROJECT_SOURCE_DIR}/utils/compilerinfo ${CMAKE_C_COMPILER} + OUTPUT_FILE ${PROJECT_BINARY_DIR}/compilerinfo.json + ) +endif() + get_property(TEST_SUITE_TARGETS GLOBAL PROPERTY TEST_SUITE_TARGETS) add_custom_target(check COMMAND ${TEST_SUITE_LIT} ${TEST_SUITE_LIT_FLAGS} . Index: utils/compilerinfo =================================================================== --- /dev/null +++ utils/compilerinfo @@ -0,0 +1,321 @@ +#!/usr/bin/env python +'''Collect information about a given compiler. Print the information as JSON +data usable for the run info section of an lnt submission.''' +# This is basically the code from lnt/testing/util/compilers.py refactored +# into a standalone tool. +import argparse +import errno +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile + + +def fatal(str): + sys.stderr.write("Fatal Error: %s\n" % str) + sys.exit(1) + + +def error(str): + sys.stderr.write("Error: %s\n" % str) + + +def _capture(args, include_stderr=False): + stderr = subprocess.PIPE + if include_stderr: + stderr = subprocess.STDOUT + try: + p = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + except OSError, e: + if e.errno == errno.ENOENT: + sys.stderr.write('no such file or directory: %r when running %s.' % + (args[0], ' '.join(args))) + return '' + out, _ = p.communicate() + return out + + +def _ishexhash(string): + return len(string) == 40 and \ + len([c + for c in string + if c.isdigit() or c in 'abcdef']) == 40 + + +def get_cc_info(path, cc_flags=[]): + """get_cc_info(path) -> { ... } + + Extract various information on the given compiler and return a dictionary of + the results.""" + + cc = path + + # Interrogate the compiler. + cc_version = _capture([cc, '-v', '-E'] + cc_flags + + ['-x', 'c', '/dev/null', '-###'], + include_stderr=True).strip() + + # Determine the assembler version, as found by the compiler. + cc_as_version = _capture([cc, "-c", '-Wa,-v', '-o', '/dev/null'] + cc_flags + + ['-x', 'assembler', '/dev/null'], + include_stderr=True).strip() + + # Determine the linker version, as found by the compiler. + tf = tempfile.NamedTemporaryFile(suffix='.c') + name = tf.name + tf.close() + tf = open(name, 'w') + print >>tf, "int main() { return 0; }" + tf.close() + cc_ld_version = _capture(([cc, "-Wl,-v", '-o', '/dev/null'] + + cc_flags + [tf.name]), + include_stderr=True).strip() + try: + os.remove(tf.name) + except OSError as e: + pass + + # Extract the default target .ll (or assembly, for non-LLVM compilers). + cc_target_assembly = _capture([cc, '-S', '-flto', '-o', '-'] + cc_flags + + ['-x', 'c', '/dev/null'], + include_stderr=True).strip() + + # Extract the compiler's response to -dumpmachine as the target. + cc_target = cc_dumpmachine = _capture([cc, '-dumpmachine']).strip() + + # Default the target to the response from dumpmachine. + cc_target = cc_dumpmachine + + # Parse out the compiler's version line and the path to the "cc1" binary. + cc1_binary = None + version_ln = None + cc_name = cc_version_num = cc_build_string = cc_extra = "" + for ln in cc_version.split('\n'): + if ' version ' in ln: + version_ln = ln + elif 'cc1' in ln or 'clang-cc' in ln: + m = re.match(r' "?([^"]*)"?.*"?-E"?.*', ln) + if not m: + fatal("unable to determine cc1 binary: %r: %r" % (cc, ln)) + cc1_binary, = m.groups() + elif "-_Amachine" in ln: + m = re.match(r'([^ ]*) *-.*', ln) + if not m: + fatal("unable to determine cc1 binary: %r: %r" % (cc, ln)) + cc1_binary, = m.groups() + if cc1_binary is None: + error("unable to find compiler cc1 binary: %r: %r" % (cc, cc_version)) + if version_ln is None: + error("unable to find compiler version: %r: %r" % (cc, cc_version)) + else: + m = re.match(r'(.*) version ([^ ]*) +(\([^(]*\))(.*)', version_ln) + if m is not None: + cc_name,cc_version_num,cc_build_string,cc_extra = m.groups() + else: + # If that didn't match, try a more basic pattern. + m = re.match(r'(.*) version ([^ ]*)', version_ln) + if m is not None: + cc_name,cc_version_num = m.groups() + else: + error("unable to determine compiler version: %r: %r" % ( + cc, version_ln)) + cc_name = "unknown" + + # Compute normalized compiler name and type. We try to grab source + # revisions, branches, and tags when possible. + cc_norm_name = None + cc_build = None + cc_src_branch = cc_alt_src_branch = None + cc_src_revision = cc_alt_src_revision = None + cc_src_tag = None + llvm_capable = False + cc_extra = cc_extra.strip() + if cc_name == 'icc': + cc_norm_name = 'icc' + cc_build = 'PROD' + cc_src_tag = cc_version_num + + elif cc_name == 'gcc' and (cc_extra == '' or + re.match(r' \(dot [0-9]+\)', cc_extra)): + cc_norm_name = 'gcc' + m = re.match(r'\(Apple Inc. build ([0-9]*)\)', cc_build_string) + if m: + cc_build = 'PROD' + cc_src_tag, = m.groups() + else: + error('unable to determine gcc build version: %r' % cc_build_string) + elif (cc_name in ('clang', 'LLVM', 'Debian clang', 'Apple clang', 'Apple LLVM') and + (cc_extra == '' or 'based on LLVM' in cc_extra or + (cc_extra.startswith('(') and cc_extra.endswith(')')))): + llvm_capable = True + if cc_name == 'Apple clang' or cc_name == 'Apple LLVM': + cc_norm_name = 'apple_clang' + else: + cc_norm_name = 'clang' + + m = re.match(r'\(([^ ]*)( ([0-9]+))?\)', cc_build_string) + if m: + cc_src_branch,_,cc_src_revision = m.groups() + + # With a CMake build, the branch is not emitted. + if cc_src_branch and not cc_src_revision and cc_src_branch.isdigit(): + cc_src_revision = cc_src_branch + cc_src_branch = "" + + # These show up with git-svn. + if cc_src_branch == '$URL$': + cc_src_branch = "" + else: + # Otherwise, see if we can match a branch and a tag name. That could + # be a git hash. + m = re.match(r'\((.+) ([^ ]+)\)', cc_build_string) + if m: + cc_src_branch,cc_src_revision = m.groups() + else: + error('unable to determine Clang development build info: %r' % ( + (cc_name, cc_build_string, cc_extra),)) + cc_src_branch = "" + + m = re.search('clang-([0-9.]*)', cc_src_branch) + if m: + cc_build = 'PROD' + cc_src_tag, = m.groups() + + # We sometimes use a tag of 9999 to indicate a dev build. + if cc_src_tag == '9999': + cc_build = 'DEV' + else: + cc_build = 'DEV' + + # Newer Clang's can report separate versions for LLVM and Clang. Parse + # the cc_extra text so we can get the maximum SVN version. + if cc_extra.startswith('(') and cc_extra.endswith(')'): + m = re.match(r'\((.+) ([^ ]+)\)', cc_extra) + if m: + cc_alt_src_branch,cc_alt_src_revision = m.groups() + + # With a CMake build, the branch is not emitted. + if cc_src_branch and not cc_src_revision and cc_src_branch.isdigit(): + cc_alt_src_revision = cc_alt_src_branch + cc_alt_src_branch = "" + + else: + error('unable to determine Clang development build info: %r' % ( + (cc_name, cc_build_string, cc_extra),)) + + elif cc_name == 'gcc' and 'LLVM build' in cc_extra: + llvm_capable = True + cc_norm_name = 'llvm-gcc' + m = re.match(r' \(LLVM build ([0-9.]+)\)', cc_extra) + if m: + llvm_build, = m.groups() + if llvm_build: + cc_src_tag = llvm_build.strip() + cc_build = 'PROD' + else: + cc_build = 'DEV' + else: + error("unable to determine compiler name: %r" % ((cc_name, + cc_build_string),)) + + if cc_build is None: + error("unable to determine compiler build: %r" % cc_version) + + # If LLVM capable, fetch the llvm target instead. + if llvm_capable: + m = re.search('target triple = "(.*)"', cc_target_assembly) + if m: + cc_target, = m.groups() + else: + error("unable to determine LLVM compiler target: %r: %r" % + (cc, cc_target_assembly)) + + cc_exec_hash = hashlib.sha1() + cc_exec_hash.update(open(cc,'rb').read()) + + info = { 'cc_build' : cc_build, + 'cc_name' : cc_norm_name, + 'cc_version_number' : cc_version_num, + 'cc_dumpmachine' : cc_dumpmachine, + 'cc_target' : cc_target, + 'cc_version' :cc_version, + 'cc_exec_hash' : cc_exec_hash.hexdigest(), + 'cc_as_version' : cc_as_version, + 'cc_ld_version' : cc_ld_version, + 'cc_target_assembly' : cc_target_assembly, + } + if cc1_binary is not None and os.path.exists(cc1_binary): + cc1_exec_hash = hashlib.sha1() + cc1_exec_hash.update(open(cc1_binary,'rb').read()) + info['cc1_exec_hash'] = cc1_exec_hash.hexdigest() + if cc_src_tag is not None: + info['cc_src_tag'] = cc_src_tag + if cc_src_revision is not None: + info['cc_src_revision'] = cc_src_revision + if cc_src_branch: + info['cc_src_branch'] = cc_src_branch + if cc_alt_src_revision is not None: + info['cc_alt_src_revision'] = cc_alt_src_revision + if cc_alt_src_branch is not None: + info['cc_alt_src_branch'] = cc_alt_src_branch + + # Infer the run order from the other things we have computed. + info['inferred_run_order'] = get_inferred_run_order(info) + + return info + +def get_inferred_run_order(info): + # If the CC has an integral src revision, use that. + if info.get('cc_src_revision', '').isdigit(): + order = int(info['cc_src_revision']) + + # If the CC has an alt src revision, use that if it is greater: + if info.get('cc_alt_src_revision','').isdigit(): + order = max(order, int(info.get('cc_alt_src_revision'))) + + return str(order) + + # Otherwise if we have a git hash, use that + if _ishexhash(info.get('cc_src_revision','')): + # If we also have an alt src revision, combine them. + # + # We don't try and support a mix of integral and hash revisions. + if _ishexhash(info.get('cc_alt_src_revision','')): + return '%s,%s' % (info['cc_src_revision'], + info['cc_alt_src_revision']) + + return info['cc_src_revision'] + + # If this is a production compiler, look for a source tag. We don't accept 0 + # or 9999 as valid source tag, since that is what llvm-gcc builds use when + # no build number is given. + if info.get('cc_build') == 'PROD': + m = re.match(r'^[0-9]+(.[0-9]+)*$', info.get('cc_src_tag','')) + if m: + return m.group(0) + + # If that failed, infer from the LLVM revision (if specified on input). + # + # FIXME: This is only used when using llvm source builds with 'lnt runtest + # nt', which itself is deprecated. We should remove this eventually. + if info.get('llvm_revision','').isdigit(): + return info['llvm_revision'] + + # Otherwise, force at least some value for run_order, as it is now generally + # required by parts of the "simple" schema. + return '0' + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('cc_executable') + p.add_argument('cflag', nargs='*', default=[]) + config = p.parse_args() + + info = get_cc_info(config.cc_executable, config.cflag) + sys.stdout.write(json.dumps(info, indent=4)) + sys.stdout.write('\n') Index: utils/lntsubmit =================================================================== --- /dev/null +++ utils/lntsubmit @@ -0,0 +1,143 @@ +#!/usr/bin/env python +'''Create a data file suitable for LNT submission. +This takes the llvm-lit json output and optionally run and machine info files +and creates a json file suitable for submission to an LNT server +(or for `lnt import`).''' +from datetime import datetime +import argparse +import json +import sys +import urllib +import urllib2 + +p = argparse.ArgumentParser() +p.add_argument('litdata') +p.add_argument('-t', '--tag', default=None, + help='Specify LNT benchmark tag (default: nts)') +p.add_argument('-n', '--name', default=None, help='Specify LNT machine name') +p.add_argument('-o', '--run-order', default=None, help='Specify LNT run order') +p.add_argument('-i', '--run-info', default=[], action='append', + metavar='FILENAME', + help='json data to put into the run info section') +p.add_argument('-m', '--machine-info', default=[], action='append', + metavar='FILENAME', + help='json data to put into machine info section') +p.add_argument('-k', '--keep-names', default=False, action='store_true', + help='Use unmodified benchmark names') +p.add_argument('-u', '--url', default=None, + help="Submit PUT request to jenkins server") +config = p.parse_args() + +# Read machine info +machine_info = dict() +for machine_info_file in config.machine_info: + machine_info_data = json.load(open(machine_info_file)) + machine_info.update(machine_info_data) + +# Construct + read run info +run_info = { + "__report_version__": "1", + "tag": "nts", +} +for run_info_file in config.run_info: + run_info_data = json.load(open(run_info_file)) + run_info.update(run_info_data) +if config.tag is not None: + run_info['tag'] = config.tag +tag = run_info['tag'] + +run_order = config.run_order +if run_order is None: + run_order = run_info.get('inferred_run_order') + if run_order is None: + sys.stderr.write("No run-order specified and no 'inferred-run-order'" + + "specified in run-info\n") + sys.exit(1) +run_info['run_order'] = run_order + +time_now = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + +# Load LIT Data +lit_data = json.load(open(config.litdata)) +tests = lit_data.get('tests') +if tests is None: + sys.stderr.write("Error: No tests in lit data\n") + sys.exit(1) + +lnt_tests = [] +for test in lit_data['tests']: + testname = test.get('name') + if testname is None: + sys.stderr.write("Warning: Skipping unnamed test\n") + continue + if not config.keep_names: + # Imitate LNTs name manipulation. + if "::" in testname: + suite_name, cc, test_name = testname.partition("::") + test_name = test_name.strip() + testname = tag + "." + test_name.rsplit('.test', 1)[0] + + metrics = test.get('metrics') + if metrics is None: + sys.stderr.write("Warning: %s has no metrics\n" % testname) + continue + lit_to_lnt_metrics = { + 'compile_time': ('.compile', float), + 'exec_time': ('.exec', float), + 'score': ('.score', float), + 'hash': ('.hash', str), + 'link_time': ('-link.compile', float), + 'size.__text': ('.code_size', float), + } + for litname, (lntname, typeconstructor) in lit_to_lnt_metrics.items(): + datum = metrics.get(litname) + if datum is None: + # Just go on, should we warn? + continue + datum = typeconstructor(datum) + result_test = { + 'Name': testname + lntname, + 'Info': {}, + 'Data': [datum] + } + lnt_tests.append(result_test) + +# Create final lnt import data structure +result = { + "Machine": { + "Info": machine_info, + "Name": config.name, + }, + "Run": { + "Info": run_info, + "Start Time": time_now, + "End Time": time_now, + }, + "Tests": lnt_tests, +} + +if not config.url: + # Just write the resulting json to stdout + sys.stdout.write(json.dumps(result, indent=4)) + sys.stdout.write("\n") +else: + # Submit the resulting data to an LNT server. + jsondata = json.dumps(result) + postdata = {'input_data': json.dumps(result), 'commit': '1'} + data = urllib.urlencode(postdata) + try: + opener = urllib2.build_opener(urllib2.HTTPHandler) + response = urllib2.urlopen(config.url, data=data) + responsedata = response.read() + + try: + resultdata = json.loads(responsedata) + result_url = resultdata.get('result_url') + if result_url is not None: + sys.stdout.write("%s\n" % result_url) + except: + sys.stderr.write("Unexpected server response:\n" + responsedata) + except urllib2.HTTPError as e: + sys.stderr.write("PUT Request failed with code %s\n" % e.code) + sys.stderr.write("%s\n" % e.reason) + sys.exit(1) Index: utils/tests/.gitignore =================================================================== --- /dev/null +++ utils/tests/.gitignore @@ -0,0 +1 @@ +/Output Index: utils/tests/compilerinfo.test =================================================================== --- /dev/null +++ utils/tests/compilerinfo.test @@ -0,0 +1,16 @@ +RUN: ../compilerinfo tools/notclang | FileCheck %s +CHECK: { +CHECK-NEXT: "inferred_run_order": "123.4.56", +CHECK-NEXT: "cc_build": "PROD", +CHECK-NEXT: "cc_version_number": "1.2.3", +CHECK-NEXT: "cc_version": "LLVM version 1.2.3 (notclang-123.4.56)\nTarget: x86_64-unknown-unknown\nInstalledDir: tools\n \\\"tools/notclang\\\" -cc1 more flags -v -E -x c /dev/null -###", +CHECK-NEXT: "cc_as_version": "", +CHECK-NEXT: "cc_ld_version": "", +CHECK-NEXT: "cc_target_assembly": "; ModuleID = '/dev/null'\nsource_filename = \"/dev/null\"\ntarget datalayout = \"e-m:o-i64:64-f80:128-n8:16:32:64-S128\"\ntarget triple = \"x86_64-unknown-unknown\"\n\n!llvm.module.flags = !{!0}\n!llvm.ident = !{!1}\n\n!0 = !{i32 1, !\"PIC Level\", i32 2}\n!1 = !{!\"LLVM version 1.2.3 (notclang-123.4.56)\"}", +CHECK-NEXT: "cc_dumpmachine": "", +CHECK-NEXT: "cc_src_tag": "123.4.56", +CHECK-NEXT: "cc_target": "x86_64-unknown-unknown", +CHECK-NEXT: "cc_src_branch": "notclang-123.4.56", +CHECK-NEXT: "cc_name": "clang", +CHECK-NEXT: "cc_exec_hash": "3d06b22714b19ea729ba52d4036472359eda73cd" +CHECK-NEXT: } Index: utils/tests/lit.cfg =================================================================== --- /dev/null +++ utils/tests/lit.cfg @@ -0,0 +1,9 @@ +import lit.formats +import sys + +config.name = 'TEST-SUITE-TOOLS' +execute_external = not sys.platform in ['win32'] +config.test_format = lit.formats.ShTest(execute_external) +config.suffixes = ['.test'] +config.test_source_root = os.path.dirname(__file__) +config.test_exec_root = config.test_source_root Index: utils/tests/lntsubmit-info.test =================================================================== --- /dev/null +++ utils/tests/lntsubmit-info.test @@ -0,0 +1,116 @@ +RUN: tail -n +2 %s | head -n 39 > %t-litdata.json && ../compilerinfo tools/notclang > %t-compilerinfo.json && ../lntsubmit --tag CoolBench -n JustATestMachine -o 123456 -i %t-compilerinfo.json -m testmachineinfo.json %t-litdata.json | FileCheck %s +{ + "__version__": [ + 0, + 6, + 0 + ], + "elapsed": 0.5209369659423828, + "tests": [ + { + "code": "PASS", + "elapsed": 0.4605419635772705, + "metrics": { + "compile_time": 0.0221, + "exec_time": 0.406, + "hash": "549fc0d04911386004eaafbf6a36ba0d", + "link_time": 0.0329, + "size": 8512, + "size.__const": 40, + "size.__cstring": 97, + "size.__la_symbol_ptr": 16, + "size.__nl_symbol_ptr": 16, + "size.__stub_helper": 36, + "size.__stubs": 12, + "size.__text": 385, + "size.__unwind_info": 80 + }, + "name": "test-suite :: SingleSource/Benchmarks/Misc/pi.test", + "output": "\n/Users/johndoe/dev/test-suite/build/tools/timeit --limit-core 0 --limit-cpu 7200 --timeout 7200 --limit-file-size 104857600 --limit-rss-size 838860800 --append-exitstatus --redirect-output /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/Output/pi.test.out --redirect-input /dev/null --summary /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/Output/pi.test.time /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/pi\n/Users/johndoe/dev/test-suite/build/tools/fpcmp -r 0.001 /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/Output/pi.test.out /Users/johndoe/dev/test-suite/SingleSource/Benchmarks/Misc/pi.reference_output" + }, + { + "code": "PASS", + "elapsed": 1.0, + "metrics": { + "exec_time": 0.5 + }, + "name": "test-suite :: simple.test" + } + ] +} +CHECK: { +CHECK-NEXT: "Machine": { +CHECK-NEXT: "Info": { +CHECK-NEXT: "OS": "MS/DOS", +CHECK-NEXT: "CPU": "i386", +CHECK-NEXT: "Memory": "640k" +CHECK-NEXT: }, +CHECK-NEXT: "Name": "JustATestMachine" +CHECK-NEXT: }, +CHECK-NEXT: "Tests": [ +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.0221 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "CoolBench.SingleSource/Benchmarks/Misc/pi.compile" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: "549fc0d04911386004eaafbf6a36ba0d" +CHECK-NEXT: ], +CHECK-NEXT: "Name": "CoolBench.SingleSource/Benchmarks/Misc/pi.hash" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.406 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "CoolBench.SingleSource/Benchmarks/Misc/pi.exec" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.0329 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "CoolBench.SingleSource/Benchmarks/Misc/pi-link.compile" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 385.0 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "CoolBench.SingleSource/Benchmarks/Misc/pi.code_size" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.5 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "CoolBench.simple.exec" +CHECK-NEXT: } +CHECK-NEXT: ], +CHECK-NEXT: "Run": { +CHECK-NEXT: "Info": { +CHECK-NEXT: "inferred_run_order": "123.4.56", +CHECK-NEXT: "__report_version__": "1", +CHECK-NEXT: "cc_build": "PROD", +CHECK-NEXT: "cc_dumpmachine": "", +CHECK-NEXT: "cc_target_assembly": "; ModuleID = '/dev/null'\nsource_filename = \"/dev/null\"\ntarget datalayout = \"e-m:o-i64:64-f80:128-n8:16:32:64-S128\"\ntarget triple = \"x86_64-unknown-unknown\"\n\n!llvm.module.flags = !{!0}\n!llvm.ident = !{!1}\n\n!0 = !{i32 1, !\"PIC Level\", i32 2}\n!1 = !{!\"LLVM version 1.2.3 (notclang-123.4.56)\"}", +CHECK-NEXT: "cc_src_tag": "123.4.56", +CHECK-NEXT: "cc_target": "x86_64-unknown-unknown", +CHECK-NEXT: "cc_src_branch": "notclang-123.4.56", +CHECK-NEXT: "cc_version_number": "1.2.3", +CHECK-NEXT: "tag": "CoolBench", +CHECK-NEXT: "cc_exec_hash": "3d06b22714b19ea729ba52d4036472359eda73cd", +CHECK-NEXT: "run_order": "123456", +CHECK-NEXT: "cc_as_version": "", +CHECK-NEXT: "cc_name": "clang", +CHECK-NEXT: "cc_ld_version": "", +CHECK-NEXT: "cc_version": "LLVM version 1.2.3 (notclang-123.4.56)\nTarget: x86_64-unknown-unknown\nInstalledDir: tools\n \\\"tools/notclang\\\" -cc1 more flags -v -E -x c /dev/null -###" +CHECK-NEXT: }, +CHECK-NEXT: "End Time": "{{.*}}", +CHECK-NEXT: "Start Time": "{{.*}}" +CHECK-NEXT: } +CHECK-NEXT: } Index: utils/tests/lntsubmit-minimal.test =================================================================== --- /dev/null +++ utils/tests/lntsubmit-minimal.test @@ -0,0 +1,84 @@ +RUN: tail -n +2 %s | head -n 31 > %t && ../lntsubmit -n JustATestMachine -o 123456 %t | FileCheck %s +{ + "__version__": [ + 0, + 6, + 0 + ], + "elapsed": 0.5209369659423828, + "tests": [ + { + "code": "PASS", + "elapsed": 0.4605419635772705, + "metrics": { + "compile_time": 0.0221, + "exec_time": 0.406, + "hash": "549fc0d04911386004eaafbf6a36ba0d", + "link_time": 0.0329, + "size": 8512, + "size.__const": 40, + "size.__cstring": 97, + "size.__la_symbol_ptr": 16, + "size.__nl_symbol_ptr": 16, + "size.__stub_helper": 36, + "size.__stubs": 12, + "size.__text": 385, + "size.__unwind_info": 80 + }, + "name": "test-suite :: SingleSource/Benchmarks/Misc/pi.test", + "output": "\n/Users/johndoe/dev/test-suite/build/tools/timeit --limit-core 0 --limit-cpu 7200 --timeout 7200 --limit-file-size 104857600 --limit-rss-size 838860800 --append-exitstatus --redirect-output /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/Output/pi.test.out --redirect-input /dev/null --summary /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/Output/pi.test.time /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/pi\n/Users/johndoe/dev/test-suite/build/tools/fpcmp -r 0.001 /Users/johndoe/dev/test-suite/build/SingleSource/Benchmarks/Misc/Output/pi.test.out /Users/johndoe/dev/test-suite/SingleSource/Benchmarks/Misc/pi.reference_output" + } + ] +} +CHECK: { +CHECK-NEXT: "Machine": { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Name": "JustATestMachine" +CHECK-NEXT: }, +CHECK-NEXT: "Tests": [ +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.0221 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "nts.SingleSource/Benchmarks/Misc/pi.compile" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: "549fc0d04911386004eaafbf6a36ba0d" +CHECK-NEXT: ], +CHECK-NEXT: "Name": "nts.SingleSource/Benchmarks/Misc/pi.hash" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.406 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "nts.SingleSource/Benchmarks/Misc/pi.exec" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 0.0329 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "nts.SingleSource/Benchmarks/Misc/pi-link.compile" +CHECK-NEXT: }, +CHECK-NEXT: { +CHECK-NEXT: "Info": {}, +CHECK-NEXT: "Data": [ +CHECK-NEXT: 385.0 +CHECK-NEXT: ], +CHECK-NEXT: "Name": "nts.SingleSource/Benchmarks/Misc/pi.code_size" +CHECK-NEXT: } +CHECK-NEXT: ], +CHECK-NEXT: "Run": { +CHECK-NEXT: "Info": { +CHECK-NEXT: "__report_version__": "1", +CHECK-NEXT: "tag": "nts", +CHECK-NEXT: "run_order": "123456" +CHECK-NEXT: }, +CHECK-NEXT: "End Time": "{{.*}}", +CHECK-NEXT: "Start Time": "{{.*}}" +CHECK-NEXT: } +CHECK-NEXT: } Index: utils/tests/testmachineinfo.json =================================================================== --- /dev/null +++ utils/tests/testmachineinfo.json @@ -0,0 +1,5 @@ +{ + "OS": "MS/DOS", + "CPU": "i386", + "Memory": "640k" +}