Index: packages/Python/lldbsuite/test/dotest.py =================================================================== --- packages/Python/lldbsuite/test/dotest.py +++ packages/Python/lldbsuite/test/dotest.py @@ -715,7 +715,22 @@ for filterspec in configuration.filters: # Optimistically set the flag to True. filtered = True - module = __import__(base) + try: + module = __import__(base) + except Exception as ex: + # We hit here when there is a failure to load in a module, + # as will happen with invalid decorators. This path happens + # when using '-f' filters. + test_filename = os.path.abspath(os.path.join(dir, name)) + if test_filename.endswith(".pyc"): + test_filename = test_filename[0:-1] + if configuration.results_formatter_object is not None: + import traceback + backtrace = traceback.format_exc() + configuration.results_formatter_object.handle_event( + EventBuilder.event_for_job_test_add_error( + test_filename, ex, backtrace)) + raise parts = filterspec.split('.') obj = module for part in parts: @@ -737,13 +752,23 @@ if configuration.filters and not filtered: continue - # Add either the filtered test case(s) (which is done before) or the entire test class. if not filterspec or not filtered: - # A simple case of just the module name. Also the failover case - # from the filterspec branch when the (base, filterspec) combo - # doesn't make sense. - configuration.suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base)) - + # Add the entire file's worth of tests since we're not filtered. + # Also the failover case when the filterspec branch + # (base, filterspec) combo doesn't make sense. + try: + configuration.suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base)) + except Exception as ex: + test_filename = os.path.abspath(os.path.join(dir, name)) + if test_filename.endswith(".pyc"): + test_filename = test_filename[0:-1] + if configuration.results_formatter_object is not None: + import traceback + backtrace = traceback.format_exc() + configuration.results_formatter_object.handle_event( + EventBuilder.event_for_job_test_add_error( + test_filename, ex, backtrace)) + raise def disabledynamics(): import lldb Index: packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park @@ -0,0 +1,13 @@ +from __future__ import print_function +from lldbsuite.test import lldbtest +from lldbsuite.test import decorators + + +class NonExistentDecoratorTestCase(lldbtest.TestBase): + + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @decorators.nonExistentDecorator(bugnumber="yt/1300") + def test(self): + """Verify non-existent decorators are picked up by test runner.""" + pass Index: packages/Python/lldbsuite/test_event/event_builder.py =================================================================== --- packages/Python/lldbsuite/test_event/event_builder.py +++ packages/Python/lldbsuite/test_event/event_builder.py @@ -321,6 +321,19 @@ return event @staticmethod + def event_for_job_test_add_error(test_filename, exception, backtrace): + event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) + event["status"] = EventBuilder.STATUS_ERROR + if test_filename is not None: + event["test_filename"] = EventBuilder._assert_is_python_sourcefile(test_filename) + if exception is not None and "__class__" in dir(exception): + event["issue_class"] = exception.__class__ + event["issue_message"] = exception + if backtrace is not None: + event["issue_backtrace"] = backtrace + return event + + @staticmethod def event_for_job_exceptional_exit( pid, worker_index, exception_code, exception_description, test_filename, command_line): Index: packages/Python/lldbsuite/test_event/formatter/pickled.py =================================================================== --- packages/Python/lldbsuite/test_event/formatter/pickled.py +++ packages/Python/lldbsuite/test_event/formatter/pickled.py @@ -33,6 +33,7 @@ def __init__(self, out_file, options): super(RawPickledFormatter, self).__init__(out_file, options) self.pid = os.getpid() + self.use_send = out_file is not None and "send" in dir(out_file) def handle_event(self, test_event): super(RawPickledFormatter, self).handle_event(test_event) @@ -50,8 +51,14 @@ # Tack on the pid. test_event["pid"] = self.pid - # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes} - import struct - msg = cPickle.dumps(test_event) - packet = struct.pack("!I%ds" % len(msg), len(msg), msg) - self.out_file.send(packet) + # If sending over a socket, we need to format this + # in a way that can be collected properly on the other + # end. + if self.use_send: + # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes} + import struct + msg = cPickle.dumps(test_event) + packet = struct.pack("!I%ds" % len(msg), len(msg), msg) + self.out_file.send(packet) + else: + cPickle.dump(test_event, self.out_file) Index: packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py @@ -0,0 +1,13 @@ +from __future__ import print_function +from lldbsuite.test import lldbtest +from lldbsuite.test import decorators + + +class NonExistentDecoratorTestCase(lldbtest.TestBase): + + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @decorators.nonExistentDecorator(bugnumber="yt/1300") + def test(self): + """Verify non-existent decorators are picked up by test runner.""" + pass Index: packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +""" +Tests that the event system reports issues during decorator +handling as errors. +""" +# System-provided imports +import os +import unittest + +# Local-provided imports +import event_collector + + +class TestCatchInvalidDecorator(unittest.TestCase): + + def test_with_whole_file(self): + """ + Test that a non-existent decorator generates a test-event error + when running all tests in the file. + """ + # Determine the test case file we're using. + test_file = os.path.join( + os.path.dirname(__file__), + os.path.pardir, + "resources", + "invalid_decorator", + "TestInvalidDecorator.py") + + # Collect all test events generated for this file. + events = event_collector.collect_events_whole_file(test_file) + + # Filter out test events. + results = [ + event + for event in events + if event.get("event", None) in ['job_result', 'test_result']] + + self.assertTrue( + len(results) > 0, + "At least one job or test result should have been returned") + + def test_with_function_filter(self): + """ + Test that a non-existent decorator generates a test-event error + when running a filtered test. + """ + # Determine the test case file we're using. + test_file_dir = os.path.join( + os.path.dirname(__file__), + os.path.pardir, + "resources", + "invalid_decorator") + + # Collect all test events generated during running of tests + # in a given directory using a test name filter. Internally, + # this runs through a different code path that needs to be + # set up to catch exceptions. + events = event_collector.collect_events_for_directory_with_filter( + test_file_dir, + "NonExistentDecoratorTestCase.test") + + # Filter out test events. + results = [ + event + for event in events + if event.get("event", None) in ['job_result', 'test_result']] + + self.assertTrue( + len(results) > 0, + "At least one job or test result should have been returned") +if __name__ == "__main__": + unittest.main() Index: packages/Python/lldbsuite/test_event/test/src/event_collector.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test_event/test/src/event_collector.py @@ -0,0 +1,83 @@ +from __future__ import absolute_import +from __future__ import print_function + +import os +import subprocess +import sys + +from six.moves import cPickle + + +def path_to_dotest_py(): + return os.path.join( + os.path.dirname(__file__), + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + "test", + "dotest.py") + + +def pickled_events_filename(): + return "/tmp/lldb_test_event_pickled_event_output.dat" + + +def _collect_events_with_command(command): + # Delete the events file if it exists + if os.path.exists(pickled_events_filename()): + os.remove(pickled_events_filename()) + + if os.path.exists(pickled_events_filename()): + raise Exception("pickled events filename should not exist!") + + # Run the single test with dotest.py, outputting + # the raw pickled events to a temp file. + with open(os.devnull, 'w') as dev_null_file: + status = subprocess.call( + command, + stdout=dev_null_file, + stderr=dev_null_file) + + # Unpickle the events + events = [] + if os.path.exists(pickled_events_filename()): + with open(pickled_events_filename(), "rb") as events_file: + while True: + try: + # print("reading event") + event = cPickle.load(events_file) + # print("read event: {}".format(event)) + if event: + events.append(event) + except EOFError: + # This is okay. + break + return events + +def collect_events_whole_file(test_filename): + command = [ + sys.executable, + path_to_dotest_py(), + "--inferior", + "--results-formatter=lldbsuite.test_event.formatter.pickled.RawPickledFormatter", + "--results-file={}".format(pickled_events_filename()), + "-p", os.path.basename(test_filename), + os.path.dirname(test_filename) + ] + return _collect_events_with_command(command) + + +def collect_events_for_directory_with_filter(test_filename, filter): + command = [ + sys.executable, + path_to_dotest_py(), + "--inferior", + "--results-formatter=lldbsuite.test_event.formatter.pickled.RawPickledFormatter", + "--results-file={}".format(pickled_events_filename()), + "-f", filter, + os.path.dirname(test_filename) + ] + return _collect_events_with_command(command)