Index: cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py =================================================================== --- cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py +++ cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py @@ -46,10 +46,8 @@ '--results-directory', type=str, metavar='', - default=os.path.abspath( - os.path.join(get_root_directory(), '..', 'results', - datetime.now().strftime('%Y-%m-%d-%H%M-%S'))), - help='directory to save results') + default=None, + help='directory to save results (default: none)') def handle_options(self, defaults): options = self.context.options @@ -86,14 +84,15 @@ 'could not find test path "{}"'.format( options.test_path)) - options.results_directory = os.path.abspath(options.results_directory) - if not os.path.isdir(options.results_directory): - try: - os.makedirs(options.results_directory, exist_ok=True) - except OSError as e: - raise Error( - 'could not create directory "{}" ({})'. - format(options.results_directory, e.strerror)) + if options.results_directory: + options.results_directory = os.path.abspath(options.results_directory) + if not os.path.isdir(options.results_directory): + try: + os.makedirs(options.results_directory, exist_ok=True) + except OSError as e: + raise Error( + 'could not create directory "{}" ({})'. + format(options.results_directory, e.strerror)) def go(self) -> ReturnCode: # noqa options = self.context.options Index: cross-project-tests/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py =================================================================== --- cross-project-tests/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py +++ cross-project-tests/debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py @@ -192,9 +192,12 @@ steps_changed = steps_str != prev_steps_str prev_steps_str = steps_str - # If this is the first pass, or something has changed, write a text - # file containing verbose information on the current status. - if current_limit == 0 or score_difference or steps_changed: + # If a results directory has been specified and this is the first + # pass or something has changed, write a text file containing + # verbose information on the current status. + if options.results_directory and (current_limit == 0 or + score_difference or + steps_changed): file_name = '-'.join( str(s) for s in [ 'status', test_name, '{{:0>{}}}'.format( @@ -231,31 +234,33 @@ current_bisect_pass_summary[pass_info[1]].append( score_difference) - per_pass_score_path = os.path.join( - options.results_directory, - '{}-per_pass_score.csv'.format(test_name)) + if options.results_directory: + per_pass_score_path = os.path.join( + options.results_directory, + '{}-per_pass_score.csv'.format(test_name)) - with open(per_pass_score_path, mode='w', newline='') as fp: - writer = csv.writer(fp, delimiter=',') - writer.writerow(['Source File', 'Pass', 'Score']) + with open(per_pass_score_path, mode='w', newline='') as fp: + writer = csv.writer(fp, delimiter=',') + writer.writerow(['Source File', 'Pass', 'Score']) - for path, pass_, score in per_pass_score: - writer.writerow([path, pass_, score]) - self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path)) + for path, pass_, score in per_pass_score: + writer.writerow([path, pass_, score]) + self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path)) - pass_summary_path = os.path.join( - options.results_directory, '{}-pass-summary.csv'.format(test_name)) + pass_summary_path = os.path.join( + options.results_directory, '{}-pass-summary.csv'.format(test_name)) - self._write_pass_summary(pass_summary_path, - current_bisect_pass_summary) + self._write_pass_summary(pass_summary_path, + current_bisect_pass_summary) def _handle_results(self) -> ReturnCode: options = self.context.options - pass_summary_path = os.path.join(options.results_directory, - 'overall-pass-summary.csv') + if options.results_directory: + pass_summary_path = os.path.join(options.results_directory, + 'overall-pass-summary.csv') - self._write_pass_summary(pass_summary_path, - self._all_bisect_pass_summary) + self._write_pass_summary(pass_summary_path, + self._all_bisect_pass_summary) return ReturnCode.OK def _clang_opt_bisect_build(self, opt_bisect_limits): Index: cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py =================================================================== --- cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py +++ cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py @@ -176,6 +176,7 @@ """Returns the path to the test results directory for the test denoted by test_name. """ + assert self.context.options.results_directory != None return os.path.join(self.context.options.results_directory, self._get_results_basename(test_name)) @@ -193,22 +194,25 @@ def _record_steps(self, test_name, steps): """Write out the set of steps out to the test's .txt and .json - results file. + results file if a results directory has been specified. """ - output_text_path = self._get_results_text_path(test_name) - with open(output_text_path, 'w') as fp: - self.context.o.auto(str(steps), stream=Stream(fp)) + if self.context.options.results_directory: + output_text_path = self._get_results_text_path(test_name) + with open(output_text_path, 'w') as fp: + self.context.o.auto(str(steps), stream=Stream(fp)) - output_dextIR_path = self._get_results_pickle_path(test_name) - with open(output_dextIR_path, 'wb') as fp: - pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL) + output_dextIR_path = self._get_results_pickle_path(test_name) + with open(output_dextIR_path, 'wb') as fp: + pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL) def _record_score(self, test_name, heuristic): - """Write out the test's heuristic score to the results .txt file. + """Write out the test's heuristic score to the results .txt file + if a results directory has been specified. """ - output_text_path = self._get_results_text_path(test_name) - with open(output_text_path, 'a') as fp: - self.context.o.auto(heuristic.verbose_output, stream=Stream(fp)) + if self.context.options.results_directory: + output_text_path = self._get_results_text_path(test_name) + with open(output_text_path, 'a') as fp: + self.context.o.auto(heuristic.verbose_output, stream=Stream(fp)) def _record_test_and_display(self, test_case): """Output test case to o stream and record test case internally for @@ -272,19 +276,20 @@ if num_tests != 0: print("@avg: ({:.4f})".format(score_sum/num_tests)) - summary_path = os.path.join(options.results_directory, 'summary.csv') - with open(summary_path, mode='w', newline='') as fp: - writer = csv.writer(fp, delimiter=',') - writer.writerow(['Test Case', 'Score', 'Error']) - - for test_case in self._test_cases: - if (test_case.score < options.fail_lt or - test_case.error is not None): - return_code = ReturnCode.FAIL - - writer.writerow([ - test_case.name, '{:.4f}'.format(test_case.score), - test_case.error - ]) + has_failed = lambda test: test.score < options.fail_lt or test.error + if any(map(has_failed, self._test_cases)): + return_code = ReturnCode.FAIL + + if options.results_directory: + summary_path = os.path.join(options.results_directory, 'summary.csv') + with open(summary_path, mode='w', newline='') as fp: + writer = csv.writer(fp, delimiter=',') + writer.writerow(['Test Case', 'Score', 'Error']) + + for test_case in self._test_cases: + writer.writerow([ + test_case.name, '{:.4f}'.format(test_case.score), + test_case.error + ]) return return_code Index: cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools/clang-opt-bisect/clang-opt-bisect-results.cpp =================================================================== --- /dev/null +++ cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools/clang-opt-bisect/clang-opt-bisect-results.cpp @@ -0,0 +1,21 @@ +// Purpose: +// Check the `clang-opt-bisect` tool runs with --results-directory. +// +// RUN: true +// RUN: %dexter_base clang-opt-bisect \ +// RUN: --debugger %dexter_regression_test_debugger \ +// RUN: --builder %dexter_regression_test_builder \ +// RUN: --cflags "%dexter_regression_test_cflags" \ +// RUN: --ldflags "%dexter_regression_test_ldflags" \ +// RUN: --results-directory=%t \ +// RUN: -- %s \ +// RUN: | FileCheck %s +// RUN: rm -rf %t +// CHECK: running pass 0 +// CHECK: wrote{{.*}}per_pass_score +// CHECK: wrote{{.*}}pass-summary +// CHECK: wrote{{.*}}overall-pass-summary + +int main() { + return 0; +} Index: cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools/clang-opt-bisect/clang-opt-bisect.cpp =================================================================== --- cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools/clang-opt-bisect/clang-opt-bisect.cpp +++ cross-project-tests/debuginfo-tests/dexter/feature_tests/subtools/clang-opt-bisect/clang-opt-bisect.cpp @@ -10,9 +10,6 @@ // RUN: -- %s \ // RUN: | FileCheck %s // CHECK: running pass 0 -// CHECK: wrote{{.*}}per_pass_score -// CHECK: wrote{{.*}}pass-summary -// CHECK: wrote{{.*}}overall-pass-summary int main() { return 0;