Add lit-style tests for the Fuzzer library

[kcc]

Summary

Add test targets and the lit-style runner.

[samsonov]

LGTM, once you address the comments. Please watch the bots after commit.

lib/Fuzzer/CMakeLists.txt

11

Put this under

if (LLVM_INCLUDE_TESTS)

lib/Fuzzer/test/fuzzer.test

1

delete empty line

12

What about ExactTest?

14

Do you need "-" at the end of this line?

lib/Fuzzer/test/lit.cfg

1

Delete this line.

8

empty line?

[kcc]

lib/Fuzzer/CMakeLists.txt

11

done

lib/Fuzzer/test/fuzzer.test

1

done

12

Hah, good catch.
In the current state the Fuzzer is not capable of solving ExactTest (nor is AFL, for example).
I have a plan of attack though.
http://reviews.llvm.org/rL227387 is part of that.

14

It's a part of output, so yes.

lib/Fuzzer/test/lit.cfg

1

done

8

done

lib/Fuzzer/CMakeLists.txt

 add_library(LLVMFuzzer STATIC
   EXCLUDE_FROM_ALL  # Do not build if you are not building fuzzers.
   FuzzerCrossOver.cpp
   FuzzerIO.cpp
   FuzzerLoop.cpp
   FuzzerMain.cpp
   FuzzerMutate.cpp
   FuzzerUtil.cpp
   )
+
+add_subdirectory(test)

lib/Fuzzer/FuzzerLoop.cpp

void Fuzzer::DeathCallback() {
   WriteToCrash(CurrentUnit, "crash-");
 }
 
 void Fuzzer::AlarmCallback() {
   size_t Seconds =
       duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
   std::cerr << "ALARM: working on the last Unit for " << Seconds << " seconds"
             << std::endl;
-  if (Seconds > 60) {
+  if (Seconds >= 3) {
     Print(CurrentUnit, "\n");
     PrintASCII(CurrentUnit, "\n");
     WriteToCrash(CurrentUnit, "timeout-");
   }
-  abort();
+  exit(1);
 }
 
 void Fuzzer::ShuffleAndMinimize() {
   if (Options.Verbosity)
     std::cerr << "Shuffle: " << Corpus.size() << "\n";
   std::vector<Unit> NewCorpus;
   random_shuffle(Corpus.begin(), Corpus.end());
   size_t MaxCov = 0;

lib/Fuzzer/test/CMakeLists.txt

+set(Tests
+  ExactTest
+  InfiniteTest
+  NullDerefTest
+  SimpleTest
+  TimeoutTest
+  )
+
+set(TestBinaries)
+
+foreach(Test ${Tests})
+  add_executable(LLVMFuzzer-${Test}
+    EXCLUDE_FROM_ALL
+    ${Test}.cpp
+    )
+  target_link_libraries(LLVMFuzzer-${Test}
+    LLVMFuzzer
+    )
+  set(TestBinaries ${TestBinaries} LLVMFuzzer-${Test})
+endforeach()
+
+set_target_properties(${TestBinaries}
+  PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  )
+
+set(EXCLUDE_FROM_ALL TRUE)
+add_lit_testsuite(check-fuzzer "Running Fuzzer tests"
+    ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${TestBinaries}
+    )
+set(EXCLUDE_FROM_ALL FALSE)
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+  )
+

lib/Fuzzer/test/fuzzer.test

+
+RUN: ./LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s --check-prefix=SimpleTest
+SimpleTest: Found the target, exiting
+
+RUN: not ./LLVMFuzzer-InfiniteTest -timeout=2 2>&1 | FileCheck %s --check-prefix=InfiniteTest
+InfiniteTest: ALARM: working on the last Unit for
+InfiniteTest-NOT: CRASHED; file written to timeout
+
+RUN: not ./LLVMFuzzer-TimeoutTest -timeout=5 2>&1 | FileCheck %s --check-prefix=TimeoutTest
+TimeoutTest: ALARM: working on the last Unit for
+TimeoutTest: CRASHED; file written to timeout
+
+RUN: not ./LLVMFuzzer-NullDerefTest 2>&1 | FileCheck %s --check-prefix=NullDerefTest
+NullDerefTest: CRASHED; file written to crash-

lib/Fuzzer/test/lit.cfg

+# import lit.util
+import lit.formats
+
+config.name = "LLVMFuzzer"
+config.test_format = lit.formats.ShTest(True)
+config.suffixes = ['.test']
+config.test_source_root = os.path.dirname(__file__)
+

lib/Fuzzer/test/lit.site.cfg.in

+config.test_exec_root = "@CMAKE_CURRENT_BINARY_DIR@"
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")