[LNT] Add support for storing hash of test binaries.

[kristof.beyls]

Summary

This commit adds support for storing hashes of the test binaries as a
new sample type. The idea is that when the hash function of the same
test is the same as on a previous run, the analyses can assume that
the binary didn't change in a meaningful way.

None of the analyses in LNT have been adapted yet to take advantage of
this extra information.

The hash value is stored as a new sample type.  Since the hash value
in many cases should not be handled the same as other sample types,
like execution time or compilation time, a distinction needs to be
made. This patch changes terminology in LNT and calls sample types
that have a meaningful order, like execution time and compilation
time, "metic fields". The hash function of a binary - as one hash
value cannot be said to be "better" than another hash value - is not
considered a "metric field".

[cmatthews]

Wow, this is wonderful. Thanks Kristof! LGTM.

I only tested this on sqlite, BTW.

[rengolin]

I can't say much about the python code, but I very much welcome this change. :)

--renato

lnt/trunk/lnt/server/db/fieldchange.py

def regenerate_fieldchanges_for_run(ts, run):
     # can be huge, and it is almost always an error to have the same rev
     # be used in so many runs.
     run_size = len(runs_to_load)
     if run_size > 50:
         warning("Generating field changes for {} runs."
                 "That will be very slow.".format(run_size))
 
     runinfo = lnt.server.reporting.analysis.RunInfo(ts, runs_to_load)
-        
-    for field in list(ts.sample_fields):
+    # Only store fieldchanges for "metric" samples like execution time;
+    # not for fields with other data, e.g. hash of a binary
+    for field in list(ts.Sample.get_metric_fields()):
         for test_id in runinfo.test_ids:
             result = runinfo.get_comparison_result(runs, previous_runs,
                                                    test_id, field)
             if result.is_result_interesting():
                 f = ts.FieldChange(start_order=start_order,
                                    end_order=run.order,
                                    test=None,
                                    machine=run.machine,

lnt/trunk/lnt/server/db/testsuitedb.py

def __init__(self, v4db, name, test_suite):
                 """
                 status_fields = set(s.status_field
                                     for s in self.Sample.fields
                                     if s.status_field is not None)
                 for field in self.Sample.fields:
                     if field not in status_fields:
                         yield field
 
-            # Dynamically create fields for all of the test suite defined sample
-            # fields.
+            @staticmethod
+            def get_metric_fields():
+                """
+                get_metric_fields() -> [SampleField*]
+
+                Get the sample fields which represent some kind of metric, i.e.
+                those which have a value that can be interpreted as better or
+                worse than other potential values for this field.
+                """
+                for field in self.Sample.fields:
+                    if field.type.name == 'Real':
+                        yield field
+
+            # Dynamically create fields for all of the test suite defined
+            # sample fields.
             #
             # FIXME: We might want to index some of these, but for a different
             # reason than above. It is possible worth it to turn the compound
             # index below into a covering index. We should evaluate this once
             # the new UI is up.
             class_dict = locals()
             for item in self.sample_fields:
                 if item.name in class_dict:
                     raise ValueError,"test suite defines reserved key %r" % (
                         name,)
 
                 if item.type.name == 'Real':
                     item.column = Column(item.name, Float)
                 elif item.type.name == 'Status':
                     item.column = Column(item.name, Integer, ForeignKey(
                             testsuite.StatusKind.id))
+                elif item.type.name == 'Hash':
+                    item.column = Column(item.name, String)
                 else:
                     raise ValueError,(
                         "test suite defines unknown sample type %r" (
                             item.type.name,))
 
                 class_dict[item.name] = item.column
 
             def __init__(self, run, test, **kwargs):

lnt/trunk/lnt/server/db/v4db.py

def __init__(self, path, config, baseline_revision=0, echo=False):
         self.xfail_status_kind = self.query(testsuite.StatusKind)\
             .filter_by(id = lnt.testing.XFAIL).first()
         assert (self.pass_status_kind and self.fail_status_kind and
                 self.xfail_status_kind), \
                 "status kinds not initialized!"
 
         # Resolve or create the known sample types.
         self.real_sample_type = self.query(testsuite.SampleType)\
-            .filter_by(name = "Real").first()
+            .filter_by(name="Real").first()
         self.status_sample_type = self.query(testsuite.SampleType)\
-            .filter_by(name = "Status").first()
-        assert (self.real_sample_type and self.status_sample_type), \
+            .filter_by(name="Status").first()
+        self.hash_sample_type = self.query(testsuite.SampleType)\
+            .filter_by(name="Hash").first()
+        assert (self.real_sample_type and self.status_sample_type and
+                self.hash_sample_type), \
             "sample types not initialized!"
 
     def close(self):
         if self.session is not None:
             self.session.close()
 
     @property
     def testsuite(self):

lnt/trunk/lnt/server/reporting/analysis.py

def get_test_status(self):
             else:
                 return REGRESSED
         else:
             if self.prev_failed:
                 return IMPROVED
             else:
                 return UNCHANGED_PASS
 
+    # FIXME: take into account hash of binary - if available. If the hash is
+    # the same, the binary is the same and therefore the difference cannot be
+    # significant - for execution time. It can be significant for compile time.
     def get_value_status(self, confidence_interval=2.576,
-                         value_precision=MIN_VALUE_PRECISION, ignore_small=True):
+                         value_precision=MIN_VALUE_PRECISION,
+                         ignore_small=True):
         if self.current is None or self.previous is None:
             return None
 
         # Don't report value errors for tests which fail, or which just started
         # passing.
         #
         # FIXME: One bug here is that we risk losing performance data on tests
         # which flop to failure then back. What would be nice to do here is to

lnt/trunk/lnt/server/reporting/dailyreport.py

class DailyReport(object):
     def __init__(self, ts, year, month, day, num_prior_days_to_include=3,
                  day_start_offset_hours=16, for_mail=False,
                  filter_machine_regex=None):
         self.ts = ts
         self.num_prior_days_to_include = num_prior_days_to_include
         self.year = year
         self.month = month
         self.day = day
-        self.fields = list(ts.Sample.get_primary_fields())
+        self.fields = list(ts.Sample.get_metric_fields())
         self.day_start_offset = datetime.timedelta(
             hours=day_start_offset_hours)
         self.for_mail = for_mail
         self.filter_machine_regex_str = filter_machine_regex
         self.filter_machine_re = None
         if self.filter_machine_regex_str:
             try:
                 self.filter_machine_re = re.compile(filter_machine_regex)
def build(self):
         #       makes it easy to identify the scope of the change.
         #
         #   (b) When a performance change occurs, it should be easy to see the
         #       performance of that test on other machines. This makes it easy
         #       to see the scope of the change and to potentially apply human
         #       discretion in determining whether or not a particular result is
         #       worth considering (as opposed to noise).
         #
-        # The idea is as follows, for each (machine, test, primary_field),
+        # The idea is as follows, for each (machine, test, metric_field),
         # classify the result into one of REGRESSED, IMPROVED, UNCHANGED_FAIL,
         # ADDED, REMOVED, PERFORMANCE_REGRESSED, PERFORMANCE_IMPROVED.
         #
         # For now, we then just aggregate by test and present the results as
         # is. This is lossless, but not nearly as nice to read as the old style
         # per-machine reports. In the future we will want to find a way to
         # combine the per-machine report style of presenting results aggregated
         # by the kind of status change, while still managing to present the

lnt/trunk/lnt/server/reporting/runs.py

def generate_run_report(run, baseurl, only_html_body=False,
     runs_to_load.add(run.id)
     if compare_to:
         runs_to_load.add(compare_to.id)
     if baseline:
         runs_to_load.add(baseline.id)
     sri = lnt.server.reporting.analysis.RunInfo(
         ts, runs_to_load, aggregation_fn, confidence_lv)
 
-    # Get the test names, primary fields and total test counts.
+    # Get the test names, metric fields and total test counts.
     test_names = ts.query(ts.Test.name, ts.Test.id).\
         order_by(ts.Test.name).\
         filter(ts.Test.id.in_(sri.test_ids)).all()
-    primary_fields = list(ts.Sample.get_primary_fields())
-    num_total_tests = len(primary_fields) * len(test_names)
+    metric_fields = list(ts.Sample.get_metric_fields())
+    num_total_tests = len(metric_fields) * len(test_names)
 
     # Gather the run-over-run changes to report, organized by field and then
     # collated by change type.
     run_to_run_info, test_results = _get_changes_by_type(
-        run, compare_to, primary_fields, test_names, num_comparison_runs, sri)
+        run, compare_to, metric_fields, test_names, num_comparison_runs, sri)
 
     # If we have a baseline, gather the run-over-baseline results and
     # changes.
     if baseline:
         run_to_baseline_info, baselined_results = _get_changes_by_type(
-            run, baseline, primary_fields, test_names, num_comparison_runs, sri)
+            run, baseline, metric_fields, test_names, num_comparison_runs, sri)
     else:
         run_to_baseline_info = baselined_results = None
 
     # Gather the run-over-run changes to report.
 
     # Collect the simplified results, if desired, for sending back to clients.
     if result is not None:
         pset_results = []
html_report = html_template.render(
         prioritized_buckets_run_over_run=prioritized_buckets_run_over_run,
         run_to_baseline_info=run_to_baseline_info,
         prioritized_buckets_run_over_baseline=prioritized_buckets_run_over_baseline,
         styles=styles,
         start_time=start_time)
 
     return subject, text_report, html_report, sri
 
-def _get_changes_by_type(run_a, run_b, primary_fields, test_names,
+
+def _get_changes_by_type(run_a, run_b, metric_fields, test_names,
                          num_comparison_runs, sri):
     comparison_results = {}
     results_by_type = []
-    for field in primary_fields:
+    for field in metric_fields:
         new_failures = []
         new_passes = []
         perf_regressions = []
         perf_improvements = []
         removed_tests = []
         added_tests = []
         existing_failures = []
         unchanged_tests = []

lnt/trunk/lnt/server/reporting/summaryreport.py

def _build_data_table(self):
                 arch = 'x86'
 
             if run_parameters['OPTFLAGS'] == '-O0':
                 build_mode = 'Debug'
             else:
                 build_mode = 'Release'
 
             # Return a datapoint for each passing field.
-            for field_name,field,status_field in ts_sample_primary_fields:
+            for field_name, field, status_field in ts_sample_metric_fields:
                 # Ignore failing samples.
                 if status_field and \
                         sample[2 + status_field.index] == lnt.testing.FAIL:
                     continue
 
                 # Ignore missing samples.
                 value = sample[2 + field.index]
                 if value is None:
def _build_data_table(self):
                 arch = 'ARM'
             elif '86' in arch:
                 arch = 'x86'
 
             # The metric is fixed.
             metric = 'Compile Time'
 
             # Report the user and wall time.
-            for field_name,field,status_field in ts_sample_primary_fields:
+            for field_name, field, status_field in ts_sample_metric_fields:
                 if field_name not in ('user_time', 'wall_time'):
                     continue
 
                 # Ignore failing samples.
                 if status_field and \
                         sample[2 + status_field.index] == lnt.testing.FAIL:
                     continue
 
def _build_data_table(self):
             # The exact datapoints in each sample depend on the testsuite
             if ts.name == 'nts':
                 return get_nts_datapoints_for_sample(ts, sample)
             else:
                 assert ts.name == 'compile'
                 return get_compile_datapoints_for_sample(ts, sample)
 
         # For each column...
-        for index,runs in enumerate(self.runs_at_index):
+        for index, runs in enumerate(self.runs_at_index):
             # For each test suite and run list...
-            for ts,(ts_runs,_) in zip(self.testsuites, runs):
+            for ts, (ts_runs, _) in zip(self.testsuites, runs):
                 ts_tests = self.tests[ts]
 
-                # Compute the primary sample fields.
-                ts_sample_primary_fields = [
+                # Compute the metric fields.
+                ts_sample_metric_fields = [
                     (f.name, f, f.status_field)
-                    for f in ts.Sample.get_primary_fields()]
+                    for f in ts.Sample.get_metric_fields()]
 
                 # Compute a mapping from run id to run.
                 run_id_map = dict((r.id, r)
                                   for r in ts_runs)
 
                 # Compute a mapping from run id to machine id.
                 run_machine_id_map = dict((r.id, r.machine.name)
                                           for r in ts_runs)

lnt/trunk/lnt/server/ui/templates/reporting/runs.html

 
 {% import "utils.html" as utils %}
 
 {#
   Macro for rendering the report for either a run-to-run display or a run-to-baseline display.
 #}
 {%
   macro add_report_changes_detail_for_field_and_bucket(
     field, show_perf, run_url, field_index, field_display_name, bucket_name, bucket, test_names,
-    primary_name, primary_field_suffix, secondary_field_suffix, secondary_info, styles
+    metric_name, metric_field_suffix, secondary_field_suffix, secondary_info, styles
   )
 %}
 
 {% if bucket and bucket_name != 'Unchanged Tests' %}
    <p>
      <table style="{{ styles['table'] }}">
        <tr>
          <th style="{{ styles['th'] }}" width="500">{{ bucket_name }} - {{ field_display_name }}</th>
        {% if not show_perf %}
          <tbody class="searchable">
          {% for name in test_names %}
            <tr><td class="benchmark-name" style="{{ styles['td'] }}">
                {{ name }}</td></tr>
          {% endfor %}
          </tbody>
        {% else %}
-         <th style="{{ styles['th'] }}">&Delta; {{ primary_field_suffix }}</th>
-         <th style="{{ styles['th'] }}">{{ primary_name }}</th>
+         <th style="{{ styles['th'] }}">&Delta; {{ metric_field_suffix }}</th>
+         <th style="{{ styles['th'] }}">{{ metric_name }}</th>
          <th style="{{ styles['th'] }}">Current</th>
-         <th style="{{ styles['th'] }}">&sigma; {{ primary_field_suffix }}</th>
+         <th style="{{ styles['th'] }}">&sigma; {{ metric_field_suffix }}</th>
          {% if secondary_info %}
          <th style="{{ styles['th'] }}">&Delta; {{ secondary_field_suffix }}</th>
          <th style="{{ styles['th'] }}">&sigma; {{ secondary_field_suffix }}</th>
          {% endif %}
          </tr>
          <tbody class="searchable">
          
          {% for name, cr, test_id in bucket %}

lnt/trunk/lnt/server/ui/templates/v4_run.html

 {% endmacro %}
 
 {% block sidebar %}
   <ul class="nav nav-list bs-docs-sidenav">
     <li><a href="#machine_info"><i class="icon-chevron-right"></i>Machine Info</a></li>
     <li><a href="#run_info"><i class="icon-chevron-right"></i>Run Info</a></li>
     <li><a href="#view_options"><i class="icon-chevron-right"></i>View Options</a></li>
     <li><a href="#report"><i class="icon-chevron-right"></i>Report</a></li>
-    {% for field in primary_fields %}
+    {% for field in metric_fields %}
       <li><a href="#{{ field.name }}"><i class="icon-chevron-right"></i>{{ field.name }}</a></li>
     {% endfor %}
   </ul>
 
   <h4>Runs:</h4>
   <ul>
   {# Show a small number of neighboring runs. #}
   {% for r in neighboring_runs %}

 
   {{request_info.html_report|safe}}
   {{ utils.render_popup_end() }}
 
   {% set graph_base=v4_url_for('v4_graph', highlight_run=run.id) %}
   <form method="GET" action="{{ graph_base }}">
 
     {# Report one table for each primary field. #}
-    {% for field in primary_fields %}
+    {% for field in metric_fields %}
       <section id="{{ field.name }}" />
       {{ utils.render_popup_begin('test_data-' + field.name, field.name, false) }}
       <table class="table table-striped table-hover table-condensed sortable">
         <thead>
           <tr>
             <th rowspan="1" class="sorttable_nosort"><input type="checkbox" id="select-all" onClick="selectAll(this)"></th>
             <th rowspan="1">Name</th>
             {% if options.show_previous %}<th>Prev</th>{% endif %}
<form method="GET" action="{{ graph_base }}">
     <p><input type="submit" value="Graph">
   </form>
 
   {% if options.show_data_table %}
     <h3>Test Data</h3>
     <table class="table table-striped table-condensed table-hover">
       <thead>
         <th>Name</th>
-        {% for field in primary_fields %}
+        {% for field in metric_fields %}
           <th colspan="3">{{field.name}}</th>
         {% endfor %}
       </thead>
       <thead>
        <th>-</th>
-        {% for field in primary_fields %}
+        {% for field in metric_fields %}
           <th>Prev</th>
           <th>Value</th>
           <th>%</th>
         {% endfor %}
       </thead>
       <tbody class="searchable">
       {% for test_name,test_id in test_info %}
         <tr>
           <td class="benchmark-name">{{ test_name }}</td>
-          {% for field in primary_fields %}
+          {% for field in metric_fields %}
         {% set cr = request_info.sri.get_run_comparison_result(run, compare_to, test_id, field) %}
             <td>{{cr.previous}}</td>
             <td>{{cr.current}}</td>
             <td>{{cr.pct_delta}}</td>
           {% endfor %}
         </tr>
       {% endfor %}
       </tbody>
     </table>
   {% endif %}
 
 {% endblock %}

lnt/trunk/lnt/server/ui/views.py

if request.args.get('json'):
                 res = sri.get_run_comparison_result(run, None, test_id, sample_field)
                 test[sample_field.name] = res.current
             json_obj['tests'][test_id] = test
 
         return flask.jsonify(**json_obj)
 
     return render_template(
         "v4_run.html", ts=ts, options=options,
-        primary_fields=list(ts.Sample.get_primary_fields()),
+        metric_fields=list(ts.Sample.get_metric_fields()),
         test_info=test_info, analysis=lnt.server.reporting.analysis,
         test_min_value_filter=test_min_value_filter,
         request_info=info)
 
 @v4_route("/order/<int:id>")
 def v4_order(id):
     # Get the testsuite.
     ts = request.get_testsuite()
return render_template("v4_graph.html", ts=ts, options=options,
                            overview_plots=overview_plots, legend=legend,
                            baseline_plots=baseline_plots)
 
 @v4_route("/global_status")
 def v4_global_status():
     from lnt.server.ui import util
 
     ts = request.get_testsuite()
-    primary_fields = sorted(list(ts.Sample.get_primary_fields()),
-                            key=lambda f: f.name)
-    fields = dict((f.name, f) for f in primary_fields)
+    metric_fields = sorted(list(ts.Sample.get_metric_fields()),
+    fields = dict((f.name, f) for f in metric_fields)
 
     # Get the latest run.
     latest = ts.query(ts.Run.start_time).\
         order_by(ts.Run.start_time.desc()).first()
 
     # If we found an entry, use that.
     if latest is not None:
         latest_date, = latest
     else:
         # Otherwise, just use today.
         latest_date = datetime.date.today()
 
     # Create a datetime for the day before the most recent run.
     yesterday = latest_date - datetime.timedelta(days=1)
 
     # Get arguments.
     revision = int(request.args.get('revision',
                                     ts.Machine.DEFAULT_BASELINE_REVISION))
-    field = fields.get(request.args.get('field', None), primary_fields[0])
+    field = fields.get(request.args.get('field', None), metric_fields[0])
 
     # Get the list of all runs we might be interested in.
     recent_runs = ts.query(ts.Run).filter(ts.Run.start_time > yesterday).all()
 
     # Aggregate the runs by machine.
     recent_runs_by_machine = util.multidict()
     for run in recent_runs:
         recent_runs_by_machine[run.machine] = run
def v4_global_status():
 
     # Order the table by worst regression.
     test_table.sort(key = lambda row: row[1], reverse=True)
 
     return render_template("v4_global_status.html",
                            ts=ts,
                            tests=test_table,
                            machines=recent_machines,
-                           fields=primary_fields,
+                           fields=metric_fields,
                            selected_field=field,
                            selected_revision=revision)
 
 @v4_route("/daily_report")
 def v4_daily_report_overview():
     # Redirect to the report for the most recent submitted run's date.
 
     ts = request.get_testsuite()

lnt/trunk/lnt/testing/__init__.py

class TestSamples:
     comparable, then the should have the same test name but different info
     paramaters.
 
     The report may include an arbitrary number of samples for each test for
     situations where the same test is run multiple times to gather statistical
     data.
     """
 
-    def __init__(self, name, data, info={}):
+    def __init__(self, name, data, info={}, conv_f=float):
         self.name = str(name)
-        self.info = dict((str(key),str(value))
-                         for key,value in info.items())
-        self.data = map(float, data)
+        self.info = dict((str(key), str(value))
+                         for key, value in info.items())
+        self.data = map(conv_f, data)
 
     def render(self):
-        return { 'Name' : self.name,
-                 'Info' : self.info,
-                 'Data' : self.data }
+        return {'Name': self.name,
+                'Info': self.info,
+                'Data': self.data}
 
-
     def __repr__(self):
         # TODO remove this
         return "TestSample({}): {} - {}".format(self.name,
                                                 self.data,
                                                 self.info)
 ###
 # Report Versioning
 

lnt/trunk/lnt/tests/nt.py

(allow file-write*      (regex #"^/private/var/tmp/")
 
     res = execute_command(test_log, basedir, args, report_dir)
 
     if res != 0:
         print >> sys.stderr, "Failure while running nightly tests!  See log: %s" % (test_log.name)
 
 # Keep a mapping of mangled test names, to the original names in the test-suite.
 TEST_TO_NAME = {}
-KNOWN_SAMPLE_KEYS = ('compile', 'exec', 'gcc.compile', 'bc.compile', 'llc.compile',
+KNOWN_SAMPLE_KEYS = ('compile', 'exec', 'hash',
+                     'gcc.compile', 'bc.compile', 'llc.compile',
                      'llc-beta.compile', 'jit.compile', 'gcc.exec', 'llc.exec',
                      'llc-beta.exec', 'jit.exec')
+
+
 def load_nt_report_file(report_path, config):
     # Compute the test samples to report.
     sample_keys = []
+
     def append_to_sample_keys(tup):
-        stat = tup[0]
+        stat = tup[1]
         assert stat in KNOWN_SAMPLE_KEYS
-        if not tup[0] in config.exclude_stat_from_submission:
+        if stat not in config.exclude_stat_from_submission:
             sample_keys.append(tup)
     if config.test_style == "simple":
         test_namespace = 'nts'
         time_stat = ''
         # for now, user time is the unqualified Time stat
         if config.test_time_stat == "real":
             time_stat = 'Real_'
-        append_to_sample_keys(('compile', 'CC_' + time_stat + 'Time', None, 'CC'))
-        append_to_sample_keys(('exec', 'Exec_' + time_stat + 'Time', None, 'Exec'))
+        append_to_sample_keys((True, 'compile', 'CC_' + time_stat + 'Time',
+                               None, 'CC', float))
+        append_to_sample_keys((False, 'hash', 'CC_Hash', None, 'CC', str))
+        append_to_sample_keys((True, 'exec', 'Exec_' + time_stat + 'Time',
+                               None, 'Exec', float))
     else:
         test_namespace = 'nightlytest'
-        append_to_sample_keys(('gcc.compile', 'GCCAS', 'time'))
-        append_to_sample_keys(('bc.compile', 'Bytecode', 'size'))
+        append_to_sample_keys((True, 'gcc.compile', 'GCCAS', 'time'))
+        append_to_sample_keys((True, 'bc.compile', 'Bytecode', 'size'))
         if config.test_llc:
-            append_to_sample_keys(('llc.compile', 'LLC compile', 'time'))
+            append_to_sample_keys((True, 'llc.compile', 'LLC compile', 'time'))
         if config.test_llcbeta:
-            append_to_sample_keys(('llc-beta.compile', 'LLC-BETA compile', 'time'))
+            append_to_sample_keys((True, 'llc-beta.compile',
+                                   'LLC-BETA compile', 'time'))
         if config.test_jit:
-            append_to_sample_keys(('jit.compile', 'JIT codegen', 'time'))
-        append_to_sample_keys(('gcc.exec', 'GCC', 'time'))
+            append_to_sample_keys((True, 'jit.compile', 'JIT codegen', 'time'))
+        append_to_sample_keys((True, 'gcc.exec', 'GCC', 'time'))
         if config.test_llc:
-            append_to_sample_keys(('llc.exec', 'LLC', 'time'))
+            append_to_sample_keys((True, 'llc.exec', 'LLC', 'time'))
         if config.test_llcbeta:
-            append_to_sample_keys(('llc-beta.exec', 'LLC-BETA', 'time'))
+            append_to_sample_keys((True, 'llc-beta.exec', 'LLC-BETA', 'time'))
         if config.test_jit:
-            append_to_sample_keys(('jit.exec', 'JIT', 'time'))
+            append_to_sample_keys((True, 'jit.exec', 'JIT', 'time'))
 
     # Load the report file.
     report_file = open(report_path, 'rb')
     reader_it = iter(csv.reader(report_file))
 
     # Get the header.
     header = reader_it.next()
     if header[0] != 'Program':
         fatal('unexpected report file, missing header')
 
     # Verify we have the keys we expect.
     if 'Program' not in header:
         fatal('missing key %r in report header' % 'Program')
     for item in sample_keys:
-        if item[1] not in header:
-            fatal('missing key %r in report header' % item[1])
+        required = item[0]
+        header_name = item[2]
+        if required and header_name not in header:
+            fatal('missing key %r in report header' % header_name)
 
     # We don't use the test info, currently.
     test_info = {}
     test_samples = []
     for row in reader_it:
         record = dict(zip(header, row))
 
         program = record['Program']
for row in reader_it:
         else:
             is_subtest = False
 
         test_base_name = '%s.%s' % (test_namespace, test_base_name)
 
         TEST_TO_NAME[test_base_name] = program_real
 
         for info in sample_keys:
-            if len(info) == 3:
-                name,key,tname = info
+            if len(info) == 4:
+                required, name, key, tname = info
                 success_key = None
+                conv_f = float
             else:
-                name,key,tname,success_key = info
+                required, name, key, tname, success_key, conv_f = info
 
             test_name = '%s.%s' % (test_base_name, name)
+            if not required and key not in record:
+                continue
             value = record[key]
             if success_key is None:
                 success_value = value
             else:
                 success_value = record[success_key]
 
             # FIXME: Move to simpler and more succinct format, using .failed.
             if success_value == '*':
for row in reader_it:
                 test_samples.append(lnt.testing.TestSamples(
                         test_name + '.success',
                         [status_value != lnt.testing.FAIL], test_info))
             else:
                 if status_value != lnt.testing.PASS:
                     test_samples.append(lnt.testing.TestSamples(
                             test_name + '.status', [status_value], test_info))
             if value != '*':
-                if tname is None:
-                    test_samples.append(lnt.testing.TestSamples(
-                            test_name, [float(value)], test_info))
-                else:
-                    test_samples.append(lnt.testing.TestSamples(
-                            test_name + '.' + tname, [float(value)], test_info))
+                sample_test_name = test_name
+                if tname is not None:
+                    sample_test_name += '.' + tname
+                    sample_test_name, [conv_f(value)], test_info,
+                    conv_f=conv_f))
 
     report_file.close()
 
     return test_samples
 
 def prepare_report_dir(config):
     # Set up the sandbox.
     sandbox_path = config.sandbox_path

 SERVER_FAIL = u'FAIL'
 SERVER_PASS = u'PASS'
 
 # Local test results names have these suffixes
 # Test will have the perf suffix if it passed, or
 # if it failed it will have a status suffix.
 LOCAL_COMPILE_PERF = "compile"
 LOCAL_COMPILE_STATUS = "compile.status"
+LOCAL_HASH = "hash"
+LOCAL_HASH_STATUS = "hash.status"
 LOCAL_EXEC_PERF = "exec"
 LOCAL_EXEC_STATUS = "exec.status"
 
 # Server results have both status and performance in each entry
 SERVER_COMPILE_RESULT = "compile_time"
 SERVER_EXEC_RESULT = "execution_time"
 SERVER_SCORE_RESULT = "score"
 SERVER_MEM_RESULT = "mem"
+SERVER_HASH_RESULT = "hash"
 
 
 class PastRunData(object):
     """To decide if we need to rerun, we must know
     what happened on each test in the first runs.
     Because the server returns data in a different format than
     the local results, this class is comprised of a per-test
     per-run aggregate of the two reports."""
for b in local_results.tests:
         if test_type == LOCAL_COMPILE_PERF:
             updating_entry.compile_time = b.data
         elif test_type == LOCAL_COMPILE_STATUS:
             updating_entry.compile_status = SERVER_FAIL
         elif test_type == LOCAL_EXEC_PERF:
             updating_entry.execution_time = b.data
         elif test_type == LOCAL_EXEC_STATUS:
             updating_entry.execution_status = SERVER_FAIL
+        elif test_type == LOCAL_HASH:
+            # do not trigger reruns just because the binary has changed.
+            pass
+        elif test_type == LOCAL_HASH_STATUS:
+            updating_entry.hash_status = SERVER_FAIL
         else:
             assert False, "Unexpected local test type."
 
         collated_results[test_name] = updating_entry
 
     # Now add on top the server results to any entry we already have.
     for full_name, results_status, perf_status in server_results:
         fields = full_name.split(".")

lnt/trunk/tests/runtest/Inputs/rerun-test-suite1/fake-report.simple.csv

-Program,CC,CC_Time,Exec,Exec_Time
-AtomicOps,pass, 0.0076,pass, 0.0003
-DefaultInitDynArrays,pass, 0.0159,pass, 0.0003
-FloatPrecision,pass, 0.0078,pass, 0.0003
-ObjC++/Hello,pass, 0.3348,pass, 0.0032
-ObjC++/property-reference,pass, 0.4295,pass, 0.0060
-ObjC++/property-reference-object,pass, 0.3569,pass, 0.0031
-ObjC++/reference-in-block-args,pass, 0.0127,pass, 0.0030
-ObjC/bitfield-access,pass, 0.0171,pass, 0.0030
-ObjC/bitfield-access-2,pass, 0.0171,pass, 0.0030
-ObjC/block-byref-aggr,pass, 0.3288,pass, 0.0030
-ObjC/constant-strings,pass, 0.0072,pass, 0.0031
-ObjC/dot-syntax,pass, 0.0223,pass, 0.0030
-ObjC/dot-syntax-1,pass, 0.0264,pass, 0.0031
-ObjC/dot-syntax-2,pass, 0.0136,pass, 0.0032
-ObjC/exceptions,pass, 0.2270,pass, 0.0032
-ObjC/exceptions-2,pass, 0.2062,pass, 0.0030
-ObjC/exceptions-3,pass, 0.2097,pass, 0.0031
-ObjC/exceptions-4,pass, 0.2103,pass, 0.0048
-ObjC/for-in,pass, 0.2147,pass, 0.0034
-ObjC/instance-method-metadata,pass, 0.2126,pass, 0.0030
-ObjC/messages,pass, 0.0193,pass, 0.0030
-ObjC/messages-2,pass, 0.0356,pass, 0.0030
-ObjC/parameter-passing,pass, 0.2268,pass, 0.0031
-ObjC/predefined-expr-in-method,pass, 0.0115,pass, 0.0031
-ObjC/property,pass, 0.2239,pass, 0.0031
-ObjC/protocols,pass, 0.0193,pass, 0.0030
-ObjC/synchronized,pass, 0.2094,pass, 0.1217
-ObjC/trivial-interface,pass, 0.2071,pass, 0.0031
-SignlessTypes/Large/cast,pass, 0.0314,pass, 0.0087
-SignlessTypes/cast-bug,pass, 0.0061,pass, 0.0003
-SignlessTypes/cast2,pass, 0.0085,pass, 0.0003
-SignlessTypes/ccc,pass, 0.0160,pass, 0.0003
-SignlessTypes/div,pass, 0.0139,pass, 0.0003
-SignlessTypes/factor,pass, 0.0169,pass, 0.0003
-SignlessTypes/rem,pass, 0.0599,pass, 0.0009
-SignlessTypes/shr,pass, 0.0139,pass, 0.0003
-StructModifyTest,pass, 0.0062,pass, 0.0003
-TestLoop,pass, 0.0088,pass, 0.0003
-Vector/SSE/sse.expandfft,pass, 0.0652,pass, 0.2459
-Vector/SSE/sse.isamax,pass, 0.0388,pass, 0.0003
-Vector/SSE/sse.shift,pass, 0.0217,pass, 0.0003
-Vector/SSE/sse.stepfft,pass, 0.0524,pass, 0.3313
-Vector/build,pass, 0.0121,pass, 0.0003
-Vector/build2,pass, 0.0159,pass, 1.1560
-Vector/divides,pass, 0.0090,pass, 0.0003
-Vector/multiplies,pass, 0.0169,pass, 1.8812
-Vector/simple,pass, 0.0134,pass, 0.0003
-Vector/sumarray,pass, 0.0099,pass, 0.0003
-Vector/sumarray-dbl,pass, 0.0107,pass, 0.0003
-block-byref-cxxobj-test,pass, 0.0148,pass, 0.0003
-block-byref-test,pass, 0.0080,pass, 0.0003
-block-call-r7674133,pass, 0.0072,pass, 0.0003
-block-copied-in-cxxobj,pass, 0.0186,pass, 0.0003
-block-copied-in-cxxobj-1,pass, 0.0165,pass, 0.0003
-blockstret,pass, 0.0089,pass, 0.0003
-byval-alignment,pass, 0.0079,pass, 0.0003
-conditional-gnu-ext,pass, 0.0066,pass, 0.0003
-conditional-gnu-ext-cxx,pass, 0.0082,pass, 0.0003
-initp1,pass, 0.0240,pass, 0.0003
-member-function-pointers,pass, 0.0120,pass, 0.0003
-ms_struct-bitfield,pass, 0.0053,pass, 0.0003
-ms_struct-bitfield-1,pass, 0.0049,pass, 0.0003
-ms_struct-bitfield-init,pass, 0.0100,pass, 0.0003
-ms_struct-bitfield-init-1,pass, 0.0119,pass, 0.0003
-ms_struct_pack_layout,pass, 0.0111,pass, 0.0003
-ms_struct_pack_layout-1,pass, 0.0046,pass, 0.0003
-printargs,pass, 0.0085,pass,0.01
-stmtexpr,pass, 0.0090, *,0.01
-vla,pass, 0.0194,pass, 0.0003
+Program,CC,CC_Time,CC_Hash,Exec,Exec_Time
+AtomicOps,pass, 0.0076,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,pass, 0.0003
+DefaultInitDynArrays,pass, 0.0159,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+FloatPrecision,pass, 0.0078,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ObjC++/Hello,pass, 0.3348,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC++/property-reference,pass, 0.4295,9b6df823b2061b2e42555d1279048b97,pass, 0.0060
+ObjC++/property-reference-object,pass, 0.3569,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC++/reference-in-block-args,pass, 0.0127,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/bitfield-access,pass, 0.0171,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/bitfield-access-2,pass, 0.0171,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/block-byref-aggr,pass, 0.3288,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/constant-strings,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/dot-syntax,pass, 0.0223,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/dot-syntax-1,pass, 0.0264,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/dot-syntax-2,pass, 0.0136,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC/exceptions,pass, 0.2270,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC/exceptions-2,pass, 0.2062,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/exceptions-3,pass, 0.2097,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/exceptions-4,pass, 0.2103,9b6df823b2061b2e42555d1279048b97,pass, 0.0048
+ObjC/for-in,pass, 0.2147,9b6df823b2061b2e42555d1279048b97,pass, 0.0034
+ObjC/instance-method-metadata,pass, 0.2126,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/messages,pass, 0.0193,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/messages-2,pass, 0.0356,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/parameter-passing,pass, 0.2268,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/predefined-expr-in-method,pass, 0.0115,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/property,pass, 0.2239,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/protocols,pass, 0.0193,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/synchronized,pass, 0.2094,9b6df823b2061b2e42555d1279048b97,pass, 0.1217
+ObjC/trivial-interface,pass, 0.2071,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+SignlessTypes/Large/cast,pass, 0.0314,9b6df823b2061b2e42555d1279048b97,pass, 0.0087
+SignlessTypes/cast-bug,pass, 0.0061,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/cast2,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/ccc,pass, 0.0160,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/div,pass, 0.0139,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/factor,pass, 0.0169,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/rem,pass, 0.0599,9b6df823b2061b2e42555d1279048b97,pass, 0.0009
+SignlessTypes/shr,pass, 0.0139,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+StructModifyTest,pass, 0.0062,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+TestLoop,pass, 0.0088,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.expandfft,pass, 0.0652,9b6df823b2061b2e42555d1279048b97,pass, 0.2459
+Vector/SSE/sse.isamax,pass, 0.0388,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.shift,pass, 0.0217,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.stepfft,pass, 0.0524,9b6df823b2061b2e42555d1279048b97,pass, 0.3313
+Vector/build,pass, 0.0121,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/build2,pass, 0.0159,9b6df823b2061b2e42555d1279048b97,pass, 1.1560
+Vector/divides,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/multiplies,pass, 0.0169,9b6df823b2061b2e42555d1279048b97,pass, 1.8812
+Vector/simple,pass, 0.0134,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/sumarray,pass, 0.0099,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/sumarray-dbl,pass, 0.0107,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-byref-cxxobj-test,pass, 0.0148,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-byref-test,pass, 0.0080,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-call-r7674133,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-copied-in-cxxobj,pass, 0.0186,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-copied-in-cxxobj-1,pass, 0.0165,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+blockstret,pass, 0.0089,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+byval-alignment,pass, 0.0079,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+conditional-gnu-ext,pass, 0.0066,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+conditional-gnu-ext-cxx,pass, 0.0082,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+initp1,pass, 0.0240,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+member-function-pointers,pass, 0.0120,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield,pass, 0.0053,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-1,pass, 0.0049,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-init,pass, 0.0100,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-init-1,pass, 0.0119,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct_pack_layout,pass, 0.0111,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct_pack_layout-1,pass, 0.0046,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+printargs,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,pass,0.01
+stmtexpr,pass, 0.0090,9b6df823b2061b2e42555d1279048b97, *,0.01
+vla,pass, 0.0194,9b6df823b2061b2e42555d1279048b97,pass, 0.0003

lnt/trunk/tests/runtest/Inputs/rerun-test-suite2/Output/ms_struct_pack_layout-1.simple.report.txt

 ---------------------------------------------------------------
 >>> ========= '/private/tmp/lnt_test_26297-2/test-2014-09-09_22-12-54/SingleSource/Benchmarks/Shootout/ms_struct_pack_layout-1' Program
 ---------------------------------------------------------------
 
 TEST-PASS: compile /private/tmp/lnt_test_26297-2/test-2014-09-09_22-12-54/SingleSource/Benchmarks/Shootout/ms_struct_pack_layout-1
 TEST-RESULT-compile-success: pass
 TEST-RESULT-compile-time: user       0.7001
+TEST-RESULT-compile-hash: 1234567890abcdef
 TEST-RESULT-compile-real-time: real       0.8219
 
 TEST-PASS: exec /private/tmp/lnt_test_26297-2/test-2014-09-09_22-12-54/SingleSource/Benchmarks/Shootout/ms_struct_pack_layout-1
 TEST-RESULT-exec-success: pass
 TEST-RESULT-exec-time: user       0.9900
-TEST-RESULT-exec-real-time: real       1.0802
+TEST-RESULT-exec-real-time: real       1.0802
 

lnt/trunk/tests/runtest/Inputs/rerun-test-suite2/Output/vla.simple.report.txt

 ---------------------------------------------------------------
 >>> ========= '/private/tmp/lnt_test_26297-2/test-2014-09-09_22-12-54/SingleSource/Benchmarks/Shootout-C++/vla' Program
 ---------------------------------------------------------------
 
 TEST-PASS: compile /private/tmp/lnt_test_26297-2/test-2014-09-09_22-12-54/SingleSource/Benchmarks/Shootout-C++/vla
 TEST-RESULT-compile-success: pass
 TEST-RESULT-compile-time: user       0.7001
+TEST-RESULT-compile-hash: 1234567890abcdef
 TEST-RESULT-compile-real-time: real       0.8219
 
 TEST-PASS: exec /private/tmp/lnt_test_26297-2/test-2014-09-09_22-12-54/SingleSource/Benchmarks/Shootout-C++/vla
 TEST-RESULT-exec-success: pass
 TEST-RESULT-exec-time: user       0.9900
 TEST-RESULT-exec-real-time: real       1.0802

lnt/trunk/tests/runtest/Inputs/rerun-test-suite2/TEST.simple.report

 }
 
 (
  ["Program"  , '\'([^\']+)\' Program'],
  [],
  ["CC"       , 'TEST-RESULT-compile-success: (pass|fail|xfail)'],
  ["CC_Time"  , 'TEST-RESULT-compile-time: user\s*([.0-9m:]+)', \&FormatTime],
  ["CC_Real_Time", 'TEST-RESULT-compile-real-time: real\s*([.0-9m:]+)', \&FormatTime],
+ ["CC_Hash",      'TEST-RESULT-compile-hash: (.*)'],
  ["Exec"     , 'TEST-RESULT-exec-success: (pass|fail|xfail)'],
  ["Exec_Time", 'TEST-RESULT-exec-time: user\s*([.0-9m:]+)', \&FormatTime],
  ["Exec_Real_Time", 'TEST-RESULT-exec-real-time: real\s*([.0-9m:]+)', \&FormatTime],
 );

lnt/trunk/tests/runtest/Inputs/rerun-test-suite2/fake-report.simple.csv

-Program,CC,CC_Time,Exec,Exec_Time
-AtomicOps,pass, 0.0076,pass, 0.0003
-DefaultInitDynArrays,pass, 0.0159,pass, 0.0003
-FloatPrecision,pass, 0.0078,pass, 0.0003
-ObjC++/Hello,pass, 0.3348,pass, 0.0032
-ObjC++/property-reference,pass, 0.4295,pass, 0.0060
-ObjC++/property-reference-object,pass, 0.3569,pass, 0.0031
-ObjC++/reference-in-block-args,pass, 0.0127,pass, 0.0030
-ObjC/bitfield-access,pass, 0.0171,pass, 0.0030
-ObjC/bitfield-access-2,pass, 0.0171,pass, 0.0030
-ObjC/block-byref-aggr,pass, 0.3288,pass, 0.0030
-ObjC/constant-strings,pass, 0.0072,pass, 0.0031
-ObjC/dot-syntax,pass, 0.0223,pass, 0.0030
-ObjC/dot-syntax-1,pass, 0.0264,pass, 0.0031
-ObjC/dot-syntax-2,pass, 0.0136,pass, 0.0032
-ObjC/exceptions,pass, 0.2270,pass, 0.0032
-ObjC/exceptions-2,pass, 0.2062,pass, 0.0030
-ObjC/exceptions-3,pass, 0.2097,pass, 0.0031
-ObjC/exceptions-4,pass, 0.2103,pass, 0.0048
-ObjC/for-in,pass, 0.2147,pass, 0.0034
-ObjC/instance-method-metadata,pass, 0.2126,pass, 0.0030
-ObjC/messages,pass, 0.0193,pass, 0.0030
-ObjC/messages-2,pass, 0.0356,pass, 0.0030
-ObjC/parameter-passing,pass, 0.2268,pass, 0.0031
-ObjC/predefined-expr-in-method,pass, 0.0115,pass, 0.0031
-ObjC/property,pass, 0.2239,pass, 0.0031
-ObjC/protocols,pass, 0.0193,pass, 0.0030
-ObjC/synchronized,pass, 0.2094,pass, 0.1217
-ObjC/trivial-interface,pass, 0.2071,pass, 0.0031
-SignlessTypes/Large/cast,pass, 0.0314,pass, 0.0087
-SignlessTypes/cast-bug,pass, 0.0061,pass, 0.0003
-SignlessTypes/cast2,pass, 0.0085,pass, 0.0003
-SignlessTypes/ccc,pass, 0.0160,pass, 0.0003
-SignlessTypes/div,pass, 0.0139,pass, 0.0003
-SignlessTypes/factor,pass, 0.0169,pass, 0.0003
-SignlessTypes/rem,pass, 0.0599,pass, 0.0009
-SignlessTypes/shr,pass, 0.0139,pass, 0.0003
-StructModifyTest,pass, 0.0062,pass, 0.0003
-TestLoop,pass, 0.0088,pass, 0.0003
-Vector/SSE/sse.expandfft,pass, 0.0652,pass, 0.2459
-Vector/SSE/sse.isamax,pass, 0.0388,pass, 0.0003
-Vector/SSE/sse.shift,pass, 0.0217,pass, 0.0003
-Vector/SSE/sse.stepfft,pass, 0.0524,pass, 0.3313
-Vector/build,pass, 0.0121,pass, 0.0003
-Vector/build2,pass, 0.0159,pass, 1.1560
-Vector/divides,pass, 0.0090,pass, 0.0003
-Vector/multiplies,pass, 0.0169,pass, 1.8812
-Vector/simple,pass, 0.0134,pass, 0.0003
-Vector/sumarray,pass, 0.0099,pass, 0.0003
-Vector/sumarray-dbl,pass, 0.0107,pass, 0.0003
-block-byref-cxxobj-test,pass, 0.0148,pass, 0.0003
-block-byref-test,pass, 0.0080,pass, 0.0003
-block-call-r7674133,pass, 0.0072,pass, 0.0003
-block-copied-in-cxxobj,pass, 0.0186,pass, 0.0003
-block-copied-in-cxxobj-1,pass, 0.0165,pass, 0.0003
-blockstret,pass, 0.0089,pass, 0.0003
-byval-alignment,pass, 0.0079,pass, 0.0003
-conditional-gnu-ext,pass, 0.0066,pass, 0.0003
-conditional-gnu-ext-cxx,pass, 0.0082,pass, 0.0003
-initp1,*, 0.0240,*, 0.0003
-member-function-pointers,pass, 0.0120,pass, 0.0003
-ms_struct-bitfield,pass, 0.0053,pass, 200.0003
-ms_struct-bitfield-1,pass, 0.0049,pass, 0.0003
-ms_struct-bitfield-init,pass, 0.0100,pass, 0.0003
-ms_struct-bitfield-init-1,pass, 0.0119,pass, 0.0003
-ms_struct_pack_layout,pass, 0.0111,pass, 0.0003
-ms_struct_pack_layout-1,pass, 0.0046,pass, 200.0003
-printargs,pass, 0.0085,*,0.02
-stmtexpr,pass, 0.0090,pass, 0.0003
-vla,pass, 0.0194,pass, 200.0003
+Program,CC,CC_Time,CC_Hash,Exec,Exec_Time
+AtomicOps,pass, 0.0076,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+DefaultInitDynArrays,pass, 0.0159,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+FloatPrecision,pass, 0.0078,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ObjC++/Hello,pass, 0.3348,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC++/property-reference,pass, 0.4295,9b6df823b2061b2e42555d1279048b97,pass, 0.0060
+ObjC++/property-reference-object,pass, 0.3569,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC++/reference-in-block-args,pass, 0.0127,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/bitfield-access,pass, 0.0171,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/bitfield-access-2,pass, 0.0171,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/block-byref-aggr,pass, 0.3288,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/constant-strings,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/dot-syntax,pass, 0.0223,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/dot-syntax-1,pass, 0.0264,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/dot-syntax-2,pass, 0.0136,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC/exceptions,pass, 0.2270,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC/exceptions-2,pass, 0.2062,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/exceptions-3,pass, 0.2097,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/exceptions-4,pass, 0.2103,9b6df823b2061b2e42555d1279048b97,pass, 0.0048
+ObjC/for-in,pass, 0.2147,9b6df823b2061b2e42555d1279048b97,pass, 0.0034
+ObjC/instance-method-metadata,pass, 0.2126,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/messages,pass, 0.0193,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/messages-2,pass, 0.0356,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/parameter-passing,pass, 0.2268,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/predefined-expr-in-method,pass, 0.0115,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/property,pass, 0.2239,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/protocols,pass, 0.0193,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/synchronized,pass, 0.2094,9b6df823b2061b2e42555d1279048b97,pass, 0.1217
+ObjC/trivial-interface,pass, 0.2071,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+SignlessTypes/Large/cast,pass, 0.0314,9b6df823b2061b2e42555d1279048b97,pass, 0.0087
+SignlessTypes/cast-bug,pass, 0.0061,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/cast2,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/ccc,pass, 0.0160,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/div,pass, 0.0139,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/factor,pass, 0.0169,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/rem,pass, 0.0599,9b6df823b2061b2e42555d1279048b97,pass, 0.0009
+SignlessTypes/shr,pass, 0.0139,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+StructModifyTest,pass, 0.0062,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+TestLoop,pass, 0.0088,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.expandfft,pass, 0.0652,9b6df823b2061b2e42555d1279048b97,pass, 0.2459
+Vector/SSE/sse.isamax,pass, 0.0388,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.shift,pass, 0.0217,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.stepfft,pass, 0.0524,9b6df823b2061b2e42555d1279048b97,pass, 0.3313
+Vector/build,pass, 0.0121,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/build2,pass, 0.0159,9b6df823b2061b2e42555d1279048b97,pass, 1.1560
+Vector/divides,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/multiplies,pass, 0.0169,9b6df823b2061b2e42555d1279048b97,pass, 1.8812
+Vector/simple,pass, 0.0134,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/sumarray,pass, 0.0099,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/sumarray-dbl,pass, 0.0107,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-byref-cxxobj-test,pass, 0.0148,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-byref-test,pass, 0.0080,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-call-r7674133,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-copied-in-cxxobj,pass, 0.0186,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-copied-in-cxxobj-1,pass, 0.0165,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+blockstret,pass, 0.0089,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+byval-alignment,pass, 0.0079,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+conditional-gnu-ext,pass, 0.0066,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+conditional-gnu-ext-cxx,pass, 0.0082,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+initp1,*, 0.0240,9b6df823b2061b2e42555d1279048b97,*, 0.0003
+member-function-pointers,pass, 0.0120,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield,pass, 0.0053,9b6df823b2061b2e42555d1279048b97,pass, 200.0003
+ms_struct-bitfield-1,pass, 0.0049,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-init,pass, 0.0100,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-init-1,pass, 0.0119,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct_pack_layout,pass, 0.0111,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct_pack_layout-1,pass, 0.0046,9b6df823b2061b2e42555d1279048b97,pass, 200.0003
+printargs,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,*,0.02
+stmtexpr,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+vla,pass, 0.0194,9b6df823b2061b2e42555d1279048b97,pass, 200.0003

lnt/trunk/tests/runtest/Inputs/test-suite/fake-report.simple.csv

-Program,CC,CC_Time,Exec,Exec_Time
-2002-04-17-PrintfChar,pass, 0.0056,pass, 0.0003
-2002-05-02-ArgumentTest,pass, 0.0058,pass, 0.0003
-2002-05-02-CastTest,pass, 0.0131,pass, 0.0003
-2002-05-02-CastTest1,pass, 0.0049,pass, 0.0003
-2002-05-02-CastTest2,pass, 0.0104,pass, 0.0003
-2002-05-02-CastTest3,pass, 0.0055,pass, 0.0003
-2002-05-02-ManyArguments,pass, 0.0090,pass, 0.0003
-2002-05-03-NotTest,pass, 0.0087,pass, 0.0003
-2002-05-19-DivTest,pass, 0.0082,pass, 0.0003
-2002-08-02-CastTest,pass, 0.0072,pass, 0.0003
-2002-08-02-CastTest2,pass, 0.0066,pass, 0.0003
-2002-08-19-CodegenBug,pass, 0.0097,pass, 0.0003
-2002-10-09-ArrayResolution,pass, 0.0077,pass, 0.0003
-2002-10-12-StructureArgs,pass, 0.0075,pass, 0.0003
-2002-10-12-StructureArgsSimple,pass, 0.0073,pass, 0.0003
-2002-10-13-BadLoad,pass, 0.0055,pass, 0.0003
-2002-12-13-MishaTest,pass, 0.0095,pass, 0.0003
-2003-04-22-Switch,pass, 0.0090,pass, 0.0003
-2003-05-02-DependentPHI,pass, 0.0083,pass, 0.0003
-2003-05-07-VarArgs,pass, 0.0166,pass, 0.0003
-2003-05-12-MinIntProblem,pass, 0.0075,pass, 0.0003
-2003-05-14-AtExit,pass, 0.0100,pass, 0.0003
-2003-05-26-Shorts,pass, 0.0100,pass, 0.0003
-2003-05-31-CastToBool,pass, 0.0119,pass, 0.0003
-2003-05-31-LongShifts,pass, 0.0081,pass, 0.0003
-2003-07-06-IntOverflow,pass, 0.0112,pass, 0.0003
-2003-07-08-BitOpsTest,pass, 0.0079,pass, 0.0003
-2003-07-09-LoadShorts,pass, 0.0122,pass, 0.0003
-2003-07-09-SignedArgs,pass, 0.0166,pass, 0.0003
-2003-07-10-SignConversions,pass, 0.0080,pass, 0.0003
-2003-08-05-CastFPToUint,pass, 0.0104,pass, 0.0003
-2003-08-11-VaListArg,pass, 0.0227,pass, 0.0003
-2003-08-20-FoldBug,pass, 0.0076,pass, 0.0003
-2003-09-18-BitFieldTest,pass, 0.0076,pass, 0.0003
-2003-10-13-SwitchTest,pass, 0.0072,pass, 0.0003
-2003-10-29-ScalarReplBug,pass, 0.0072,pass, 0.0003
-2004-02-02-NegativeZero,pass, 0.0056,pass, 0.0003
-2004-06-20-StaticBitfieldInit,pass, 0.0072,pass, 0.0003
-2004-11-28-GlobalBoolLayout,pass, 0.0108,pass, 0.0003
-2005-05-11-Popcount-ffs-fls,pass, 0.0200,pass, 0.0003
-2005-05-12-Int64ToFP,pass, 0.0085,pass, 0.0003
-2005-05-13-SDivTwo,pass, 0.0084,pass, 0.0003
-2005-07-15-Bitfield-ABI,pass, 0.0072,pass, 0.0003
-2005-07-17-INT-To-FP,pass, 0.0107,pass, 0.0005
-2005-11-29-LongSwitch,pass, 0.0078,pass, 0.0003
-2006-01-23-UnionInit,pass, 0.0129,pass, 0.0003
-2006-01-29-SimpleIndirectCall,pass, 0.0075,pass, 0.0003
-2006-02-04-DivRem,pass, 0.0084,pass, 0.0003
-2006-12-01-float_varg,pass, 0.0070,pass, 0.0003
-2006-12-04-DynAllocAndRestore,pass, 0.0069,pass, 0.0003
-2006-12-07-Compare64BitConstant,pass, 0.0115,pass, 0.0003
-2006-12-11-LoadConstants,pass, 0.0474,pass, 0.0003
-2007-01-04-KNR-Args,pass, 0.0086,pass, 0.0003
-2007-03-02-VaCopy,pass, 0.0088,pass, 0.0003
-2007-04-10-BitfieldTest,pass, 0.0094,pass, 0.0003
-2008-04-18-LoopBug,pass, 0.0082,pass, 0.0003
-2008-04-20-LoopBug2,pass, 0.0129,pass, 0.0003
-2008-07-13-InlineSetjmp,pass, 0.0079,pass, 0.0003
-2009-04-16-BitfieldInitialization,pass, 0.0114,pass, 0.0003
-2009-12-07-StructReturn,pass, 0.0073,pass, 0.0003
-2010-05-24-BitfieldTest,pass, 0.0074,pass, 0.0003
-AtomicOps,pass, 0.0076,pass, 0.0003
-DefaultInitDynArrays,pass, 0.0159,pass, 0.0003
-FloatPrecision,pass, 0.0078,pass, 0.0003
-ObjC++/Hello,pass, 0.3348,pass, 0.0032
-ObjC++/property-reference,pass, 0.4295,pass, 0.0060
-ObjC++/property-reference-object,pass, 0.3569,pass, 0.0031
-ObjC++/reference-in-block-args,pass, 0.0127,pass, 0.0030
-ObjC/bitfield-access,pass, 0.0171,pass, 0.0030
-ObjC/bitfield-access-2,pass, 0.0171,pass, 0.0030
-ObjC/block-byref-aggr,pass, 0.3288,pass, 0.0030
-ObjC/constant-strings,pass, 0.0072,pass, 0.0031
-ObjC/dot-syntax,pass, 0.0223,pass, 0.0030
-ObjC/dot-syntax-1,pass, 0.0264,pass, 0.0031
-ObjC/dot-syntax-2,pass, 0.0136,pass, 0.0032
-ObjC/exceptions,pass, 0.2270,pass, 0.0032
-ObjC/exceptions-2,pass, 0.2062,pass, 0.0030
-ObjC/exceptions-3,pass, 0.2097,pass, 0.0031
-ObjC/exceptions-4,pass, 0.2103,pass, 0.0048
-ObjC/for-in,pass, 0.2147,pass, 0.0034
-ObjC/instance-method-metadata,pass, 0.2126,pass, 0.0030
-ObjC/messages,pass, 0.0193,pass, 0.0030
-ObjC/messages-2,pass, 0.0356,pass, 0.0030
-ObjC/parameter-passing,pass, 0.2268,pass, 0.0031
-ObjC/predefined-expr-in-method,pass, 0.0115,pass, 0.0031
-ObjC/property,pass, 0.2239,pass, 0.0031
-ObjC/protocols,pass, 0.0193,pass, 0.0030
-ObjC/synchronized,pass, 0.2094,pass, 0.1217
-ObjC/trivial-interface,pass, 0.2071,pass, 0.0031
-SignlessTypes/Large/cast,pass, 0.0314,pass, 0.0087
-SignlessTypes/cast-bug,pass, 0.0061,pass, 0.0003
-SignlessTypes/cast2,pass, 0.0085,pass, 0.0003
-SignlessTypes/ccc,pass, 0.0160,pass, 0.0003
-SignlessTypes/div,pass, 0.0139,pass, 0.0003
-SignlessTypes/factor,pass, 0.0169,pass, 0.0003
-SignlessTypes/rem,pass, 0.0599,pass, 0.0009
-SignlessTypes/shr,pass, 0.0139,pass, 0.0003
-StructModifyTest,pass, 0.0062,pass, 0.0003
-TestLoop,pass, 0.0088,pass, 0.0003
-Vector/SSE/sse.expandfft,pass, 0.0652,pass, 0.2459
-Vector/SSE/sse.isamax,pass, 0.0388,pass, 0.0003
-Vector/SSE/sse.shift,pass, 0.0217,pass, 0.0003
-Vector/SSE/sse.stepfft,pass, 0.0524,pass, 0.3313
-Vector/build,pass, 0.0121,pass, 0.0003
-Vector/build2,pass, 0.0159,pass, 1.1560
-Vector/divides,pass, 0.0090,pass, 0.0003
-Vector/multiplies,pass, 0.0169,pass, 1.8812
-Vector/simple,pass, 0.0134,pass, 0.0003
-Vector/sumarray,pass, 0.0099,pass, 0.0003
-Vector/sumarray-dbl,pass, 0.0107,pass, 0.0003
-block-byref-cxxobj-test,pass, 0.0148,pass, 0.0003
-block-byref-test,pass, 0.0080,pass, 0.0003
-block-call-r7674133,pass, 0.0072,pass, 0.0003
-block-copied-in-cxxobj,pass, 0.0186,pass, 0.0003
-block-copied-in-cxxobj-1,pass, 0.0165,pass, 0.0003
-blockstret,pass, 0.0089,pass, 0.0003
-byval-alignment,pass, 0.0079,pass, 0.0003
-conditional-gnu-ext,pass, 0.0066,pass, 0.0003
-conditional-gnu-ext-cxx,pass, 0.0082,pass, 0.0003
-initp1,pass, 0.0240,pass, 0.0003
-member-function-pointers,pass, 0.0120,pass, 0.0003
-ms_struct-bitfield,pass, 0.0053,pass, 0.0003
-ms_struct-bitfield-1,pass, 0.0049,pass, 0.0003
-ms_struct-bitfield-init,pass, 0.0100,pass, 0.0003
-ms_struct-bitfield-init-1,pass, 0.0119,pass, 0.0003
-ms_struct_pack_layout,pass, 0.0111,pass, 0.0003
-ms_struct_pack_layout-1,pass, 0.0046,pass, 0.0003
-printargs,pass, 0.0085,pass, 0.0003
-stmtexpr,pass, 0.0090,pass, 0.0003
-vla,pass, 0.0194,pass, 0.0003
+Program,CC,CC_Time,CC_Hash,Exec,Exec_Time
[phone]-PrintfChar,pass, 0.0056,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-ArgumentTest,pass, 0.0058,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastTest,pass, 0.0131,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastTest1,pass, 0.0049,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastTest2,pass, 0.0104,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastTest3,pass, 0.0055,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-ManyArguments,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-NotTest,pass, 0.0087,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-DivTest,pass, 0.0082,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastTest,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastTest2,pass, 0.0066,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CodegenBug,pass, 0.0097,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-ArrayResolution,pass, 0.0077,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-StructureArgs,pass, 0.0075,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-StructureArgsSimple,pass, 0.0073,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-BadLoad,pass, 0.0055,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-MishaTest,pass, 0.0095,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-Switch,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-DependentPHI,pass, 0.0083,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-VarArgs,pass, 0.0166,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-MinIntProblem,pass, 0.0075,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-AtExit,pass, 0.0100,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-Shorts,pass, 0.0100,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastToBool,pass, 0.0119,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-LongShifts,pass, 0.0081,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-IntOverflow,pass, 0.0112,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-BitOpsTest,pass, 0.0079,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-LoadShorts,pass, 0.0122,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-SignedArgs,pass, 0.0166,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-SignConversions,pass, 0.0080,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-CastFPToUint,pass, 0.0104,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-VaListArg,pass, 0.0227,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-FoldBug,pass, 0.0076,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-BitFieldTest,pass, 0.0076,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-SwitchTest,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-ScalarReplBug,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-NegativeZero,pass, 0.0056,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-StaticBitfieldInit,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-GlobalBoolLayout,pass, 0.0108,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-Popcount-ffs-fls,pass, 0.0200,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-Int64ToFP,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-SDivTwo,pass, 0.0084,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-Bitfield-ABI,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-INT-To-FP,pass, 0.0107,9b6df823b2061b2e42555d1279048b97,pass, 0.0005
[phone]-LongSwitch,pass, 0.0078,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-UnionInit,pass, 0.0129,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-SimpleIndirectCall,pass, 0.0075,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-DivRem,pass, 0.0084,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-float_varg,pass, 0.0070,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-DynAllocAndRestore,pass, 0.0069,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-Compare64BitConstant,pass, 0.0115,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-LoadConstants,pass, 0.0474,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-KNR-Args,pass, 0.0086,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-VaCopy,pass, 0.0088,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-BitfieldTest,pass, 0.0094,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-LoopBug,pass, 0.0082,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-LoopBug2,pass, 0.0129,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-InlineSetjmp,pass, 0.0079,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-BitfieldInitialization,pass, 0.0114,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-StructReturn,pass, 0.0073,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
[phone]-BitfieldTest,pass, 0.0074,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+AtomicOps,pass, 0.0076,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+DefaultInitDynArrays,pass, 0.0159,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+FloatPrecision,pass, 0.0078,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ObjC++/Hello,pass, 0.3348,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC++/property-reference,pass, 0.4295,9b6df823b2061b2e42555d1279048b97,pass, 0.0060
+ObjC++/property-reference-object,pass, 0.3569,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC++/reference-in-block-args,pass, 0.0127,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/bitfield-access,pass, 0.0171,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/bitfield-access-2,pass, 0.0171,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/block-byref-aggr,pass, 0.3288,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/constant-strings,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/dot-syntax,pass, 0.0223,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/dot-syntax-1,pass, 0.0264,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/dot-syntax-2,pass, 0.0136,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC/exceptions,pass, 0.2270,9b6df823b2061b2e42555d1279048b97,pass, 0.0032
+ObjC/exceptions-2,pass, 0.2062,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/exceptions-3,pass, 0.2097,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/exceptions-4,pass, 0.2103,9b6df823b2061b2e42555d1279048b97,pass, 0.0048
+ObjC/for-in,pass, 0.2147,9b6df823b2061b2e42555d1279048b97,pass, 0.0034
+ObjC/instance-method-metadata,pass, 0.2126,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/messages,pass, 0.0193,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/messages-2,pass, 0.0356,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/parameter-passing,pass, 0.2268,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/predefined-expr-in-method,pass, 0.0115,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/property,pass, 0.2239,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+ObjC/protocols,pass, 0.0193,9b6df823b2061b2e42555d1279048b97,pass, 0.0030
+ObjC/synchronized,pass, 0.2094,9b6df823b2061b2e42555d1279048b97,pass, 0.1217
+ObjC/trivial-interface,pass, 0.2071,9b6df823b2061b2e42555d1279048b97,pass, 0.0031
+SignlessTypes/Large/cast,pass, 0.0314,9b6df823b2061b2e42555d1279048b97,pass, 0.0087
+SignlessTypes/cast-bug,pass, 0.0061,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/cast2,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/ccc,pass, 0.0160,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/div,pass, 0.0139,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/factor,pass, 0.0169,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+SignlessTypes/rem,pass, 0.0599,9b6df823b2061b2e42555d1279048b97,pass, 0.0009
+SignlessTypes/shr,pass, 0.0139,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+StructModifyTest,pass, 0.0062,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+TestLoop,pass, 0.0088,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.expandfft,pass, 0.0652,9b6df823b2061b2e42555d1279048b97,pass, 0.2459
+Vector/SSE/sse.isamax,pass, 0.0388,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.shift,pass, 0.0217,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/SSE/sse.stepfft,pass, 0.0524,9b6df823b2061b2e42555d1279048b97,pass, 0.3313
+Vector/build,pass, 0.0121,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/build2,pass, 0.0159,9b6df823b2061b2e42555d1279048b97,pass, 1.1560
+Vector/divides,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/multiplies,pass, 0.0169,9b6df823b2061b2e42555d1279048b97,pass, 1.8812
+Vector/simple,pass, 0.0134,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/sumarray,pass, 0.0099,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+Vector/sumarray-dbl,pass, 0.0107,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-byref-cxxobj-test,pass, 0.0148,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-byref-test,pass, 0.0080,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-call-r7674133,pass, 0.0072,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-copied-in-cxxobj,pass, 0.0186,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+block-copied-in-cxxobj-1,pass, 0.0165,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+blockstret,pass, 0.0089,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+byval-alignment,pass, 0.0079,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+conditional-gnu-ext,pass, 0.0066,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+conditional-gnu-ext-cxx,pass, 0.0082,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+initp1,pass, 0.0240,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+member-function-pointers,pass, 0.0120,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield,pass, 0.0053,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-1,pass, 0.0049,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-init,pass, 0.0100,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct-bitfield-init-1,pass, 0.0119,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct_pack_layout,pass, 0.0111,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+ms_struct_pack_layout-1,pass, 0.0046,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+printargs,pass, 0.0085,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+stmtexpr,pass, 0.0090,9b6df823b2061b2e42555d1279048b97,pass, 0.0003
+vla,pass, 0.0194,9b6df823b2061b2e42555d1279048b97,pass, 0.0003

lnt/trunk/tests/runtest/nt.py

 # RUN: lnt runtest nt \
 # RUN:   --sandbox %t.SANDBOX \
 # RUN:   --test-suite %S/Inputs/test-suite \
 # RUN:   --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:   --no-timestamp --no-configure > %t.log 2> %t.err
 # RUN: FileCheck --check-prefix CHECK-NOCONF < %t.err %s
 # CHECK-NOCONF-NOT: configuring
 #
+# Check a basic nt run on a test-suite without binary hash support.
+# RUN: lnt runtest nt \
+# RUN:   --sandbox %t.SANDBOX-NO-HASH \
+# RUN:   --test-suite %S/Inputs/test-suite-nohash \
+# RUN:   --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
+# RUN:   --no-timestamp > %t.log 2> %t.err
+# RUN: FileCheck --check-prefix CHECK-STDOUT < %t.log %s
+# RUN: FileCheck --check-prefix CHECK-BASIC < %t.err %s
+# RUN: FileCheck --check-prefix CHECK-REPORT < %t.SANDBOX-NO-HASH/build/report.json %s
+#
 # Manually set a run order.
 # RUN: lnt runtest nt \
 # RUN:   --sandbox %t.SANDBOX \
 # RUN:   --test-suite %S/Inputs/test-suite \
 # RUN:   --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
 # RUN:   --no-timestamp --run-order=123 > %t.log 2> %t.err
 # RUN: FileCheck --check-prefix CHECK-RESULTS < %t.SANDBOX/build/report.json %s
 # CHECK-RESULTS: "run_order": "123"