[lld][benchmark] Be tolerant of optional metric units in output

[int3]

Summary

... and set up a test scaffolding so we can verify the behavior.

It seems that the perf on my machine does emit the msec unit for
task-clock, hence this fix.

Note that perf stat has an -x flag that will output things in an
easy-to-parse CSV-like format, but that flag will cause things like
the 'seconds time elapsed' data to be omitted, which is why I didn't
take that route here.

P.S. I'm barely familiar with CMake, so I just copied the stuff from
clang/bindings/python/tests/CMakeLists.txt which also has a python unit
test.

[compnerd]

lld/test/CMakeLists.txt

39

This doesn't seem correct - the check-* shouldn't be a dependency for running tests.

[int3]

lld/test/CMakeLists.txt

39

True, but neither is LLDUnitTests, and I was following its lead...

Looking at llvm/CMakeLists.txt, it seems that over there they define a LLVM_ADDITIONAL_TEST_TARGETS variable that gets added as a dependency to the check-all target, alongside the test dependencies for the main test suite; I guess we could do something similar here.

[int3]

clean up cmake

lld/test/CMakeLists.txt

if (NOT LLD_BUILT_STANDALONE)
   list(APPEND LLD_TEST_DEPS
     FileCheck count llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres
     llvm-dis llvm-dwarfdump llvm-lib llvm-lipo llvm-mc llvm-nm llvm-objcopy
     llvm-objdump llvm-pdbutil llvm-readelf llvm-readobj llvm-strip
     llvm-symbolizer not obj2yaml opt split-file yaml2obj
     )
 endif()
 
+set(LLD_ADDITIONAL_TESTS "")
 if (LLVM_INCLUDE_TESTS)
-  list(APPEND LLD_TEST_DEPS LLDUnitTests)
+  list(APPEND LLD_ADDITIONAL_TESTS LLDUnitTests)
+  list(APPEND LLD_ADDITIONAL_TESTS check-lld-benchmark)
 endif()
 
 add_lit_testsuite(check-lld "Running lld test suite"
   ${CMAKE_CURRENT_BINARY_DIR}
-  DEPENDS ${LLD_TEST_DEPS}
+  DEPENDS ${LLD_TEST_DEPS} ${LLD_ADDITIONAL_TESTS}
   )
 
 add_custom_target(lld-test-depends DEPENDS ${LLD_TEST_DEPS})
 set_target_properties(lld-test-depends PROPERTIES FOLDER "lld tests")
 
 add_lit_testsuites(LLD ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${LLD_TEST_DEPS}
   )
 
 set_target_properties(check-lld PROPERTIES FOLDER "lld tests")
 
 # Add a legacy target spelling: lld-test
 add_custom_target(lld-test)
 add_dependencies(lld-test check-lld)
 set_target_properties(lld-test PROPERTIES FOLDER "lld tests")

lld/unittests/CMakeLists.txt

 add_custom_target(LLDUnitTests)
 set_target_properties(LLDUnitTests PROPERTIES FOLDER "lld tests")
 
 set(CMAKE_BUILD_WITH_INSTALL_RPATH OFF)
 
 # add_lld_unittest(test_dirname file1.cpp file2.cpp)
 #
 # Will compile the list of files together and link against lld
 # Produces a binary named 'basename(test_dirname)'.
 function(add_lld_unittest test_dirname)
   add_unittest(LLDUnitTests ${test_dirname} ${ARGN})
   target_link_libraries(${test_dirname} ${LLVM_COMMON_LIBS})
 endfunction()
 
 add_subdirectory(DriverTests)
 add_subdirectory(MachOTests)
+
+add_custom_target(check-lld-benchmark
+  COMMAND ${CMAKE_COMMAND} -E env
+          PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../utils
+          "${Python3_EXECUTABLE}" -m unittest test_benchmark.py
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+set_target_properties(check-lld-benchmark PROPERTIES FOLDER "lld tests")

lld/unittests/test_benchmark.py

+import unittest
+
+import benchmark
+
+
+class TestBenchmark(unittest.TestCase):
+    def test_parse_perf_run(self):
+        with_units = b"""
+ Performance counter stats for 'ld64.lld.darwinnew @response.txt':
+
+          4,307.46 msec task-clock                #    0.999 CPUs utilized
+                70      context-switches          #    0.016 K/sec
+                 1      cpu-migrations            #    0.000 K/sec
+            53,770      page-faults               #    0.012 M/sec
+   <not supported>      cycles
+   <not supported>      instructions
+   <not supported>      branches
+   <not supported>      branch-misses
+
+       4.310496496 seconds time elapsed
+
+       4.048564000 seconds user
+       0.258972000 seconds sys
+"""
+
+        without_units = b"""
+ Performance counter stats for 'ld64.lld.darwinnew @response.txt':
+
+          4,307.46      task-clock                #    0.999 CPUs utilized
+                70      context-switches          #    0.016 K/sec
+                 1      cpu-migrations            #    0.000 K/sec
+            53,770      page-faults               #    0.012 M/sec
+   <not supported>      cycles
+   <not supported>      instructions
+   <not supported>      branches
+   <not supported>      branch-misses
+
+       4.310496496 seconds time elapsed
+
+       4.048564000 seconds user
+       0.258972000 seconds sys
+"""
+
+        expected = {
+            "seconds-elapsed": 4.310496496,
+            "task-clock": 4307.46,
+            "context-switches": 70,
+            "cpu-migrations": 1,
+            "page-faults": 53770,
+        }
+
+        self.assertEqual(benchmark.parsePerf(with_units), expected)
+        self.assertEqual(benchmark.parsePerf(without_units), expected)

lld/utils/benchmark.py

 parser.add_argument('--runs', type=int, default=10)
 parser.add_argument('--lld', help='Path to the lld binary', default='../ld.lld')
 parser.add_argument('--wrapper', default='')
 parser.add_argument('--machine', required=True)
 parser.add_argument('--revision', required=True)
 parser.add_argument('--threads', action='store_true')
 parser.add_argument('--url', help='The lnt server url to send the results to',
                     default='http://localhost:8000/db_default/v4/link/submitRun')
-args = parser.parse_args()
+global args
 
 class Bench:
     def __init__(self, directory, variant):
         self.directory = directory
         self.variant = variant
     def __str__(self):
         if not self.variant:
             return self.directory
def parsePerfNum(num):
     except ValueError:
         return float(num)
 
 def parsePerfLine(line):
     ret = {}
     line = line.split(b'#')[0].strip()
     if len(line) != 0:
         p = line.split()
-        ret[p[1].strip().decode('ascii')] = parsePerfNum(p[0])
+        metric_name = p[1 if len(p) < 3 else 2].strip()
+        ret[metric_name.decode('ascii')] = parsePerfNum(p[0])
     return ret
 
 def parsePerf(output):
     ret = {}
     lines = [x.strip() for x in output.split(b'\n')]
 
     seconds = [x for x in lines if b'seconds time elapsed' in x][0]
     seconds = seconds.strip().split()[0].strip()
ret = {
         'tests' : tests
     }
     return json.dumps(ret, sort_keys=True, indent=4)
 
 def submitToServer(data):
     data2 = urlencode({ 'input_data' : data }).encode('ascii')
     urlopen(Request(args.url, data2))
 
+if __name__ == '__main__':
+    args = parser.parse_args()
     os.chdir(args.benchmark_directory)
     data = buildLntJson(getBenchmarks())
     submitToServer(data)