Index: packages/Python/lldbsuite/test/dotest.py =================================================================== --- packages/Python/lldbsuite/test/dotest.py +++ packages/Python/lldbsuite/test/dotest.py @@ -676,73 +676,98 @@ # This is to locate the lldb.py module. Insert it right after sys.path[0]. sys.path[1:1] = [lldbPythonDir] + +def visit_file(dir, name): + # Try to match the regexp pattern, if specified. + if configuration.regexp: + import re + if not re.search(configuration.regexp, name): + # We didn't match the regex, we're done. + return + + # We found a match for our test. Add it to the suite. + + # Update the sys.path first. + if not sys.path.count(dir): + sys.path.insert(0, dir) + base = os.path.splitext(name)[0] + + # Thoroughly check the filterspec against the base module and admit + # the (base, filterspec) combination only when it makes sense. + filterspec = None + for filterspec in configuration.filters: + # Optimistically set the flag to True. + filtered = True + module = __import__(base) + parts = filterspec.split('.') + obj = module + for part in parts: + try: + parent, obj = obj, getattr(obj, part) + except AttributeError: + # The filterspec has failed. + filtered = False + break + + # If filtered, we have a good filterspec. Add it. + if filtered: + # print("adding filter spec %s to module %s" % (filterspec, module)) + configuration.suite.addTests( + unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)) + continue + + # Forgo this module if the (base, filterspec) combo is invalid + if configuration.filters and not filtered: + return + + if not filterspec or not filtered: + # Add the entire file's worth of tests since we're not filtered. + # Also the fail-over case when the filterspec branch + # (base, filterspec) combo doesn't make sense. + configuration.suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base)) + + def visit(prefix, dir, names): """Visitor function for os.path.walk(path, visit, arg).""" dir_components = set(dir.split(os.sep)) excluded_components = set(['.svn', '.git']) if dir_components.intersection(excluded_components): - #print("Detected an excluded dir component: %s" % dir) return - for name in names: - if '.py' == os.path.splitext(name)[1] and name.startswith(prefix): + # Gather all the Python test file names that follow the Test*.py pattern. + python_test_files = [ + name + for name in names + if name.endswith('.py') and name.startswith(prefix)] + # Visit all the python test files. + for name in python_test_files: + try: + # Ensure we error out if we have multiple tests with the same + # base name. + # Future improvement: find all the places where we work with base + # names and convert to full paths. We have directory structure + # to disambiguate these, so we shouldn't need this constraint. if name in configuration.all_tests: raise Exception("Found multiple tests with the name %s" % name) configuration.all_tests.add(name) - # Try to match the regexp pattern, if specified. - if configuration.regexp: - import re - if re.search(configuration.regexp, name): - #print("Filename: '%s' matches pattern: '%s'" % (name, regexp)) - pass - else: - #print("Filename: '%s' does not match pattern: '%s'" % (name, regexp)) - continue - - # We found a match for our test. Add it to the suite. - - # Update the sys.path first. - if not sys.path.count(dir): - sys.path.insert(0, dir) - base = os.path.splitext(name)[0] - - # Thoroughly check the filterspec against the base module and admit - # the (base, filterspec) combination only when it makes sense. - filterspec = None - for filterspec in configuration.filters: - # Optimistically set the flag to True. - filtered = True - module = __import__(base) - parts = filterspec.split('.') - obj = module - for part in parts: - try: - parent, obj = obj, getattr(obj, part) - except AttributeError: - # The filterspec has failed. - filtered = False - break - - # If filtered, we have a good filterspec. Add it. - if filtered: - #print("adding filter spec %s to module %s" % (filterspec, module)) - configuration.suite.addTests( - unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)) - continue - - # Forgo this module if the (base, filterspec) combo is invalid - if configuration.filters and not filtered: - continue - - # Add either the filtered test case(s) (which is done before) or the entire test class. - if not filterspec or not filtered: - # A simple case of just the module name. Also the failover case - # from the filterspec branch when the (base, filterspec) combo - # doesn't make sense. - configuration.suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base)) + # Run the relevant tests in the python file. + visit_file(dir, name) + except Exception as ex: + # Convert this exception to a test event error for the file. + test_filename = os.path.abspath(os.path.join(dir, name)) + if configuration.results_formatter_object is not None: + # Grab the backtrace for the exception. + import traceback + backtrace = traceback.format_exc() + + # Generate the test event. + configuration.results_formatter_object.handle_event( + EventBuilder.event_for_job_test_add_error( + test_filename, ex, backtrace)) + raise def disabledynamics(): Index: packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park @@ -0,0 +1,13 @@ +from __future__ import print_function +from lldbsuite.test import lldbtest +from lldbsuite.test import decorators + + +class NonExistentDecoratorTestCase(lldbtest.TestBase): + + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @decorators.nonExistentDecorator(bugnumber="yt/1300") + def test(self): + """Verify non-existent decorators are picked up by test runner.""" + pass Index: packages/Python/lldbsuite/test_event/event_builder.py =================================================================== --- packages/Python/lldbsuite/test_event/event_builder.py +++ packages/Python/lldbsuite/test_event/event_builder.py @@ -321,6 +321,19 @@ return event @staticmethod + def event_for_job_test_add_error(test_filename, exception, backtrace): + event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) + event["status"] = EventBuilder.STATUS_ERROR + if test_filename is not None: + event["test_filename"] = EventBuilder._assert_is_python_sourcefile(test_filename) + if exception is not None and "__class__" in dir(exception): + event["issue_class"] = exception.__class__ + event["issue_message"] = exception + if backtrace is not None: + event["issue_backtrace"] = backtrace + return event + + @staticmethod def event_for_job_exceptional_exit( pid, worker_index, exception_code, exception_description, test_filename, command_line): Index: packages/Python/lldbsuite/test_event/formatter/pickled.py =================================================================== --- packages/Python/lldbsuite/test_event/formatter/pickled.py +++ packages/Python/lldbsuite/test_event/formatter/pickled.py @@ -33,6 +33,7 @@ def __init__(self, out_file, options): super(RawPickledFormatter, self).__init__(out_file, options) self.pid = os.getpid() + self.use_send = out_file is not None and "send" in dir(out_file) def handle_event(self, test_event): super(RawPickledFormatter, self).handle_event(test_event) @@ -50,8 +51,14 @@ # Tack on the pid. test_event["pid"] = self.pid - # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes} - import struct - msg = cPickle.dumps(test_event) - packet = struct.pack("!I%ds" % len(msg), len(msg), msg) - self.out_file.send(packet) + # If sending over a socket, we need to format this + # in a way that can be collected properly on the other + # end. + if self.use_send: + # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes} + import struct + msg = cPickle.dumps(test_event) + packet = struct.pack("!I%ds" % len(msg), len(msg), msg) + self.out_file.send(packet) + else: + cPickle.dump(test_event, self.out_file) Index: packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py @@ -0,0 +1,13 @@ +from __future__ import print_function +from lldbsuite.test import lldbtest +from lldbsuite.test import decorators + + +class NonExistentDecoratorTestCase(lldbtest.TestBase): + + mydir = lldbtest.TestBase.compute_mydir(__file__) + + @decorators.nonExistentDecorator(bugnumber="yt/1300") + def test(self): + """Verify non-existent decorators are picked up by test runner.""" + pass Index: packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +""" +Tests that the event system reports issues during decorator +handling as errors. +""" +# System-provided imports +import os +import unittest + +# Local-provided imports +import event_collector + + +class TestCatchInvalidDecorator(unittest.TestCase): + + TEST_DIR = os.path.join( + os.path.dirname(__file__), + os.path.pardir, + "resources", + "invalid_decorator") + + def test_with_whole_file(self): + """ + Test that a non-existent decorator generates a test-event error + when running all tests in the file. + """ + # Determine the test case file we're using. + test_file = os.path.join(self.TEST_DIR, "TestInvalidDecorator.py") + + # Collect all test events generated for this file. + error_results = _filter_error_results( + event_collector.collect_events_whole_file(test_file)) + + self.assertGreater( + len(error_results), + 0, + "At least one job or test error result should have been returned") + + def test_with_function_filter(self): + """ + Test that a non-existent decorator generates a test-event error + when running a filtered test. + """ + # Collect all test events generated during running of tests + # in a given directory using a test name filter. Internally, + # this runs through a different code path that needs to be + # set up to catch exceptions. + error_results = _filter_error_results( + event_collector.collect_events_for_directory_with_filter( + self.TEST_DIR, + "NonExistentDecoratorTestCase.test")) + + self.assertGreater( + len(error_results), + 0, + "At least one job or test error result should have been returned") + + +def _filter_error_results(events): + # Filter out job result events. + return [ + event + for event in events + if event.get("event", None) in ["job_result", "test_result"] and + event.get("status", None) == "error" + ] + + +if __name__ == "__main__": + unittest.main() Index: packages/Python/lldbsuite/test_event/test/src/event_collector.py =================================================================== --- /dev/null +++ packages/Python/lldbsuite/test_event/test/src/event_collector.py @@ -0,0 +1,85 @@ +from __future__ import absolute_import +from __future__ import print_function + +import os +import subprocess +import sys +import tempfile + +# noinspection PyUnresolvedReferences +from six.moves import cPickle + + +def path_to_dotest_py(): + return os.path.join( + os.path.dirname(__file__), + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + "test", + "dotest.py") + + +def _make_pickled_events_filename(): + with tempfile.NamedTemporaryFile( + prefix="lldb_test_event_pickled_event_output", + delete=False) as temp_file: + return temp_file.name + + +def _collect_events_with_command(command, events_filename): + # Run the single test with dotest.py, outputting + # the raw pickled events to a temp file. + with open(os.devnull, 'w') as dev_null_file: + subprocess.call( + command, + stdout=dev_null_file, + stderr=dev_null_file) + + # Unpickle the events + events = [] + if os.path.exists(events_filename): + with open(events_filename, "rb") as events_file: + while True: + try: + # print("reading event") + event = cPickle.load(events_file) + # print("read event: {}".format(event)) + if event: + events.append(event) + except EOFError: + # This is okay. + break + os.remove(events_filename) + return events + + +def collect_events_whole_file(test_filename): + events_filename = _make_pickled_events_filename() + command = [ + sys.executable, + path_to_dotest_py(), + "--inferior", + "--results-formatter=lldbsuite.test_event.formatter.pickled.RawPickledFormatter", + "--results-file={}".format(events_filename), + "-p", os.path.basename(test_filename), + os.path.dirname(test_filename) + ] + return _collect_events_with_command(command, events_filename) + + +def collect_events_for_directory_with_filter(test_filename, filter_desc): + events_filename = _make_pickled_events_filename() + command = [ + sys.executable, + path_to_dotest_py(), + "--inferior", + "--results-formatter=lldbsuite.test_event.formatter.pickled.RawPickledFormatter", + "--results-file={}".format(events_filename), + "-f", filter_desc, + os.path.dirname(test_filename) + ] + return _collect_events_with_command(command, events_filename)