[test-suite] Update litsupport/module/microbenchmark.py to report individual timing results from 1 test.

[homerdin]

Summary

These changes are dependent on the changes purposed in https://reviews.llvm.org/D43314.

With the purposed changes to lit we can report the individual timing results for each microbenchmark with one test. This updates the litsupport plugin to make further use of the benchmark library.

I have also changed the current microbenchmarks that use this plugin by removing the individual tests from the XRay microbenchmarks as they should no longer be needed. Please let me know if this causes any problems with the XRay tests uses @eizan .

With the purposed changes example output is:

Terminal:

PASS: test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench.test (1 of 1)
********** TEST 'test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench.test' RESULTS **********
MicroBenchmarks: 6 
compile_time: 0.6487 
hash: "daec0d37414da26fdae56f0d9bfa2cc0" 
link_time: 0.0637 
**********
*** MICRO-TEST 'BM_RDTSCP_Cost RESULTS ***
    exec_time:  12.7087 
*** MICRO-TEST 'BM_ReturnInstrumentedPatched RESULTS ***
    exec_time:  9.0223 
*** MICRO-TEST 'BM_ReturnInstrumentedPatchedThenUnpatched RESULTS ***
    exec_time:  2.6744 
*** MICRO-TEST 'BM_ReturnInstrumentedPatchedWithLogHandler RESULTS ***
    exec_time:  36.5216 
*** MICRO-TEST 'BM_ReturnInstrumentedUnPatched RESULTS ***
    exec_time:  2.6745 
*** MICRO-TEST 'BM_ReturnNeverInstrumented RESULTS ***
    exec_time:  2.0074 
**********

JSON output:

"tests": [
  {
    "code": "PASS",
    "elapsed": null,
    "metrics": {
      "exec_time": 2.67445
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench/BM_ReturnInstrumentedUnPatched.test",
    "output": ""
  },
  {
    "code": "PASS",
    "elapsed": null,
    "metrics": {
      "exec_time": 2.67435
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench/BM_ReturnInstrumentedPatchedThenUnpatched.test",
    "output": ""
  },
  {
    "code": "PASS",
    "elapsed": null,
    "metrics": {
      "exec_time": 36.5216
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench/BM_ReturnInstrumentedPatchedWithLogHandler.test",
    "output": ""
  },
  {
    "code": "PASS",
    "elapsed": null,
    "metrics": {
      "exec_time": 12.7087
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench/BM_RDTSCP_Cost.test",
    "output": ""
  },
  {
    "code": "PASS",
    "elapsed": null,
    "metrics": {
      "exec_time": 2.00737
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench/BM_ReturnNeverInstrumented.test",
    "output": ""
  },
  {
    "code": "PASS",
    "elapsed": null,
    "metrics": {
      "exec_time": 9.02227
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench/BM_ReturnInstrumentedPatched.test",
    "output": ""
  },
  {
    "code": "PASS",
    "elapsed": 5.6960108280181885,
    "metrics": {
      "MicroBenchmarks": 6,
      "compile_time": 0.6487,
      "hash": "daec0d37414da26fdae56f0d9bfa2cc0",
      "link_time": 0.0637
    },
    "name": "test-suite :: MicroBenchmarks/XRay/ReturnReference/retref-bench.test",
    "output": "\nMicroBenchmarks/XRay/ReturnReference/retref-bench --benchmark_format=csv > MicroBenchmarks/XRay/ReturnReference/Output/retref-bench.test.bench.csv"
  }

[homerdin]

Including full context

[homerdin]

Raising the lit.local.cfg into /Microbenchmark from comment in https://reviews.llvm.org/D43319

[rengolin]

LGTM after D43314 goes in. Thanks!

MicroBenchmarks/CMakeLists.txt

+file(COPY lit.local.cfg DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
 add_subdirectory(libs)
 add_subdirectory(XRay)
+add_subdirectory(LCALS)

MicroBenchmarks/XRay/FDRMode/CMakeLists.txt

 check_cxx_compiler_flag(-fxray-instrument COMPILER_HAS_FXRAY_INSTRUMENT)
 if(ARCH STREQUAL "x86" AND COMPILER_HAS_FXRAY_INSTRUMENT)
-  file(COPY lit.local.cfg DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-
   list(APPEND CPPFLAGS -std=c++11 -Wl,--gc-sections -fxray-instrument)
   list(APPEND LDFLAGS -fxray-instrument)
-  llvm_test_run(--benchmark_filter=dummy_skip_ignore)
+  llvm_test_run()
   llvm_test_executable(fdrmode-bench fdrmode-bench.cc)
   target_link_libraries(fdrmode-bench benchmark)
-
-  file(COPY fdrmode-bench_BM_XRayFDRMultiThreaded_1_thread.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY fdrmode-bench_BM_XRayFDRMultiThreaded_2_thread.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY fdrmode-bench_BM_XRayFDRMultiThreaded_4_thread.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY fdrmode-bench_BM_XRayFDRMultiThreaded_8_thread.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY fdrmode-bench_BM_XRayFDRMultiThreaded_16_thread.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY fdrmode-bench_BM_XRayFDRMultiThreaded_32_thread.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
 endif()

MicroBenchmarks/XRay/FDRMode/fdrmode-bench_BM_XRayFDRMultiThreaded_16_thread.test

-RUN: %S/fdrmode-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_XRayFDRMultiThreaded/threads:16$

MicroBenchmarks/XRay/FDRMode/fdrmode-bench_BM_XRayFDRMultiThreaded_1_thread.test

-RUN: %S/fdrmode-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_XRayFDRMultiThreaded/threads:1$

MicroBenchmarks/XRay/FDRMode/fdrmode-bench_BM_XRayFDRMultiThreaded_2_thread.test

-RUN: %S/fdrmode-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_XRayFDRMultiThreaded/threads:2$

MicroBenchmarks/XRay/FDRMode/fdrmode-bench_BM_XRayFDRMultiThreaded_32_thread.test

-RUN: %S/fdrmode-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_XRayFDRMultiThreaded/threads:32$

MicroBenchmarks/XRay/FDRMode/fdrmode-bench_BM_XRayFDRMultiThreaded_4_thread.test

-RUN: %S/fdrmode-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_XRayFDRMultiThreaded/threads:4$

MicroBenchmarks/XRay/FDRMode/fdrmode-bench_BM_XRayFDRMultiThreaded_8_thread.test

-RUN: %S/fdrmode-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_XRayFDRMultiThreaded/threads:8$

MicroBenchmarks/XRay/FDRMode/lit.local.cfg

-config.environment['XRAY_OPTIONS'] = 'patch_premain=false xray_naive_log=false xray_fdr_log=true'
-test_modules = config.test_modules
-if 'run' in test_modules:
-    # Insert microbenchmark module behind 'run'
-    test_modules.insert(test_modules.index('run')+1, 'microbenchmark')
-    # Timeit results are not useful for microbenchmarks
-    if 'timeit' in test_modules:
-        test_modules.remove('timeit')

MicroBenchmarks/XRay/ReturnReference/CMakeLists.txt

 check_cxx_compiler_flag(-fxray-instrument COMPILER_HAS_FXRAY_INSTRUMENT)
 if(ARCH STREQUAL "x86" AND COMPILER_HAS_FXRAY_INSTRUMENT)
-  file(COPY lit.local.cfg DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-
   list(APPEND CPPFLAGS -std=c++11 -Wl,--gc-sections -fxray-instrument)
   list(APPEND LDFLAGS -fxray-instrument)
-  llvm_test_run(--benchmark_filter=dummy_skip_ignore)
+  llvm_test_run()
   llvm_test_executable(retref-bench retref-bench.cc)
   target_link_libraries(retref-bench benchmark)
-
-  file(COPY retref-bench_BM_ReturnNeverInstrumented.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY retref-bench_BM_ReturnInstrumentedUnPatched.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY retref-bench_BM_ReturnInstrumentedPatchedThenUnpatched.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY retref-bench_BM_ReturnInstrumentedPatched.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY retref-bench_BM_RDTSCP_Cost.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-  file(COPY retref-bench_BM_ReturnInstrumentedPatchedWithLogHandler.test
-       DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
 endif()

MicroBenchmarks/XRay/ReturnReference/lit.local.cfg

-config.environment['XRAY_OPTIONS'] = 'patch_premain=false xray_naive_log=false'
-test_modules = config.test_modules
-if 'run' in test_modules:
-    # Insert microbenchmark module behind 'run'
-    test_modules.insert(test_modules.index('run')+1, 'microbenchmark')
-    # Timeit results are not useful for microbenchmarks
-    if 'timeit' in test_modules:
-        test_modules.remove('timeit')

MicroBenchmarks/XRay/ReturnReference/retref-bench_BM_RDTSCP_Cost.test

-RUN: %S/retref-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_RDTSCP_Cost

MicroBenchmarks/XRay/ReturnReference/retref-bench_BM_ReturnInstrumentedPatched.test

-RUN: %S/retref-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_ReturnInstrumentedPatched

MicroBenchmarks/XRay/ReturnReference/retref-bench_BM_ReturnInstrumentedPatchedThenUnpatched.test

-RUN: %S/retref-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_ReturnInstrumentedPatchedThenUnpatched

MicroBenchmarks/XRay/ReturnReference/retref-bench_BM_ReturnInstrumentedPatchedWithLogHandler.test

-RUN: %S/retref-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_ReturnInstrumentedPatchedWithLogHandler

MicroBenchmarks/XRay/ReturnReference/retref-bench_BM_ReturnInstrumentedUnPatched.test

-RUN: %S/retref-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_ReturnInstrumentedUnPatched

MicroBenchmarks/XRay/ReturnReference/retref-bench_BM_ReturnNeverInstrumented.test

-RUN: %S/retref-bench --benchmark_repetitions=10 --benchmark_report_aggregates_only=true --benchmark_filter=BM_ReturnNeverInstrumented

litsupport/modules/microbenchmark.py

def _mutateCommandLine(context, commandline):
     return cmd.toCommandline()
 
 
 def _mutateScript(context, script):
     return testplan.mutateScript(context, script, _mutateCommandLine)
 
 
 def _collectMicrobenchmarkTime(context, microbenchfiles):
-    result = 0.0
     for f in microbenchfiles:
         with open(f) as inp:
             lines = csv.reader(inp)
             # First line: "name,iterations,real_time,cpu_time,time_unit..."
             for line in lines:
                 if line[0] == 'name':
                     continue
-                # Note that we cannot create new tests here, so for now we just
-                # add up all the numbers here.
-                result += float(line[3])
-    return {'microbenchmark_time_ns': lit.Test.toMetricValue(result)}
+                # Name for MicroBenchmark
+                name = line[0]
+                # Create Result object with PASS
+                microBenchmark = lit.Test.Result(lit.Test.PASS)
+
+                # Index 3 is cpu_time
+                microBenchmark.addMetric('exec_time', lit.Test.toMetricValue(float(line[3])))
+              
+                # Add Micro Result 
+                context.micro_results[name] = microBenchmark
+
+    # returning the number of microbenchmarks collected as a metric for the base test
+    return ({'MicroBenchmarks': lit.Test.toMetricValue(len(context.micro_results))})
 
 
 def mutatePlan(context, plan):
     context.microbenchfiles = []
     plan.runscript = _mutateScript(context, plan.runscript)
     plan.metric_collectors.append(
         lambda context: _collectMicrobenchmarkTime(context,
                                                    context.microbenchfiles)
     )

litsupport/testplan.py

def _executePlan(context, plan):
     return lit.Test.PASS
 
 
 def executePlanTestResult(context, testplan):
     """Convenience function to invoke _executePlan() and construct a
     lit.test.Result() object for the results."""
     context.result_output = ""
     context.result_metrics = {}
+    context.micro_results = {}
 
     result_code = _executePlan(context, testplan)
 
     # Build test result object
     result = lit.Test.Result(result_code, context.result_output)
     for key, value in context.result_metrics.items():
         result.addMetric(key, value)
+    for key, value in context.micro_results.items():
+        result.addMicroResult(key, value)
+
     return result
 
 
 def check_output(commandline, *aargs, **dargs):
     """Wrapper around subprocess.check_output that logs the command."""
     logging.info(" ".join(commandline))
     return subprocess.check_output(commandline, *aargs, **dargs)
 
 
 def check_call(commandline, *aargs, **dargs):
     """Wrapper around subprocess.check_call that logs the command."""
     logging.info(" ".join(commandline))
     return subprocess.check_call(commandline, *aargs, **dargs)