Added expectedFlakey test decorator

[vharron]

Summary

Flakey tests get two chances to pass

[clayborg]

Looks good.

[vharron]

Added expectedFlakeyClang/Gcc, switched a bunch of tests to use new decorator.

test/expression_command/call-function/TestCallStopAndContinue.py

def setUp(self):
         # Find the line number to break for main.c.
         self.line = line_number('main.cpp',
                                 '// Please test these expressions while stopped at this line:')
         self.func_line = line_number ('main.cpp', 
                                 '{ 5, "five" }')
 
     @skipUnlessDarwin
     @dsym_test
-    @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+    @expectedFlakeyDarwin("llvm.org/pr20274")
     def test_with_dsym(self):
         """Test gathering result from interrupted function call."""
         self.buildDsym()
         self.call_function()
 
     @dwarf_test
-    @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX
+    @expectedFlakeyDarwin("llvm.org/pr20274")
     def test_with_dwarf(self):
         """Test gathering result from interrupted function call."""
         self.buildDwarf()
         self.call_function()
 
     def call_function(self):
         """Test gathering result from interrupted function call."""
         self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

test/expression_command/call-restarts/TestCallThatRestarts.py

class ExprCommandThatRestartsTestCase(TestBase):
     @skipIfDarwin # llvm.org/pr19246: intermittent failure
     def test_with_dsym(self):
         """Test calling std::String member function."""
         self.buildDsym()
         self.call_function()
 
     @dwarf_test
     @skipIfFreeBSD # llvm.org/pr19246: intermittent failure
-    @expectedFailureLinux("llvm.org/pr19246") # intermittent failure
+    @expectedFailureLinux("llvm.org/pr19246")
     @skipIfDarwin # llvm.org/pr19246: intermittent failure
     @skipIfWindows # Test relies on signals, unsupported on Windows
     def test_with_dwarf(self):
         """Test calling std::String member function."""
         self.buildDwarf()
         self.call_function()
 
     def check_after_call (self, num_sigchld):

test/expression_command/timeout/TestCallWithTimeout.py

class ExprCommandWithTimeoutsTestCase(TestBase):
     @dsym_test
     @expectedFailureDarwin # failed 1/134 runs, line 83, value.IsValid() 
     def test_with_dsym(self):
         """Test calling std::String member function."""
         self.buildDsym()
         self.call_function()
 
     @expectedFailureFreeBSD("llvm.org/pr19605") # fails on buildbot
-    @expectedFailureLinux("llvm.org/pr20275") # fails intermittently on Linux
+    @expectedFlakeyLinux("llvm.org/pr20275")
     @dwarf_test
     def test_with_dwarf(self):
         """Test calling std::String member function."""
         self.buildDwarf()
         self.call_function()
 
     def call_function(self):
         """Test calling function with timeout."""

test/functionalities/attach_resume/TestAttachResume.py

class AttachResumeTestCase(TestBase):
     @expectedFailureFreeBSD('llvm.org/pr19310')
     @skipIfRemote
     @dwarf_test
     def test_attach_continue_interrupt_detach(self):
         """Test attach/continue/interrupt/detach"""
         self.buildDwarf()
         self.process_attach_continue_interrupt_detach()
 
-    @expectedFailureLinux('llvm.org/pr19478') # intermittent ~2/14 runs
+    @expectedFlakeyLinux('llvm.org/pr19478') # intermittent ~2/14 runs
     @skipIfRemote
     def process_attach_continue_interrupt_detach(self):
         """Test attach/continue/interrupt/detach"""
 
         exe = os.path.join(os.getcwd(), exe_name)
 
         popen = self.spawnSubprocess(exe)
         self.addTearDownHook(self.cleanupSubprocesses)

test/functionalities/avoids-fd-leak/TestFdLeak.py

def do_test (self, commands):
 
         process = target.LaunchSimple (None, None, self.get_process_working_directory())
         self.assertTrue(process, PROCESS_IS_VALID)
 
         self.assertTrue(process.GetState() == lldb.eStateExited, "Process should have exited.")
         self.assertTrue(process.GetExitStatus() == 0,
                 "Process returned non-zero status. Were incorrect file descriptors passed?")
 
-    @expectedFailure(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
-    @expectedFailureLinux # xfail flakey test to get buildbot green
+    @expectedFlakey(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd
+    @expectedFlakeyLinux
     @skipIfWindows # The check for descriptor leakage needs to be implemented differently here.
     @skipIfTargetAndroid() # Android have some other file descriptors open by the shell
     def test_fd_leak_multitarget (self):
         self.buildDefault()
         exe = os.path.join (os.getcwd(), "a.out")
 
         target = self.dbg.CreateTarget(exe)
         breakpoint = target.BreakpointCreateBySourceRegex ('Set breakpoint here', lldb.SBFileSpec ("main.c", False))

test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py

class ExprDoesntDeadlockTestCase(TestBase):
     @dsym_test
     def test_with_dsym_and_run_command(self):
         """Test that expr will time out and allow other threads to run if it blocks - with dsym."""
         self.buildDsym()
         self.expr_doesnt_deadlock()
 
     @dwarf_test
     @expectedFailureFreeBSD('llvm.org/pr17946')
-    @expectedFailureLinux # failed 1/365 test runs, line 61, thread.IsValid()
+    @expectedFlakeyLinux # failed 1/365 test runs, line 61, thread.IsValid()
     def test_with_dwarf_and_run_command(self):
         """Test that expr will time out and allow other threads to run if it blocks."""
         self.buildDwarf()
         self.expr_doesnt_deadlock()
 
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)

test/functionalities/stop-hook/TestStopHookMechanism.py

class StopHookMechanismTestCase(TestBase):
     @skipUnlessDarwin
     @dsym_test
     def test_with_dsym(self):
         """Test the stop-hook mechanism."""
         self.buildDsym()
         self.stop_hook_firing()
 
     @skipIfFreeBSD # llvm.org/pr15037
-    @expectedFailureLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
+    @expectedFlakeyLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @dwarf_test
     def test_with_dwarf(self):
         """Test the stop-hook mechanism."""
         self.buildDwarf()
         self.stop_hook_firing()
 
     def setUp(self):

test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py

class StopHookForMultipleThreadsTestCase(TestBase):
     def test_stop_hook_multiple_threads_with_dsym(self):
         """Test that lldb stop-hook works for multiple threads."""
         self.buildDsym(dictionary=self.d)
         self.setTearDownCleanup(dictionary=self.d)
         self.stop_hook_multiple_threads()
 
     @dwarf_test
     @expectedFailureFreeBSD("llvm.org/pr15037")
-    @expectedFailureLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
+    @expectedFlakeyLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux
     @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows")
     @skipIfFreeBSD # llvm.org/pr22784
     def test_stop_hook_multiple_threads_with_dwarf(self):
         """Test that lldb stop-hook works for multiple threads."""
         self.buildDwarf(dictionary=self.d)
         self.setTearDownCleanup(dictionary=self.d)
         self.stop_hook_multiple_threads()
 

test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py

class CreateAfterAttachTestCase(TestBase):
     def test_create_after_attach_with_dsym(self):
         """Test thread creation after process attach."""
         self.buildDsym(dictionary=self.getBuildFlags(use_cpp11=False))
         self.create_after_attach(use_fork=False)
 
     @skipIfFreeBSD # Hangs.  May be the same as Linux issue llvm.org/pr16229 but
                    # not yet investigated.  Revisit once required functionality
                    # is implemented for FreeBSD.
-    @expectedFailureLinux("llvm.org/pr16229") # Occasionally hangs on the build bot
-    @skipIfLinux                              # Occasionally hangs on the build bot
+    @expectedFlakeyLinux("llvm.org/pr16229")
+    @skipIfLinux # Occasionally hangs on the build bot, expectedFailureLinux
     @dwarf_test
     def test_create_after_attach_with_dwarf_and_popen(self):
         """Test thread creation after process attach."""
         self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False))
         self.create_after_attach(use_fork=False)
 
     @skipIfFreeBSD # Hangs. Revisit once required functionality is implemented
                    # for FreeBSD.
     @dwarf_test
     @skipIfRemote
-    @expectedFailureLinux # this test fails 1/100 dosep runs
+    @expectedFlakeyLinux # this test fails 1/100 dosep runs
     def test_create_after_attach_with_dwarf_and_fork(self):
         """Test thread creation after process attach."""
         self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False))
         self.create_after_attach(use_fork=True)
 
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)

test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py

class ThreadSpecificBreakTestCase(TestBase):
     def test_with_dsym_python(self):
         """Test that we obey thread conditioned breakpoints."""
         self.buildDsym()
         self.do_thread_specific_break()
 
     @expectedFailureFreeBSD('llvm.org/pr18522') # hits break in another thread in testrun
     @python_api_test
     @dwarf_test
-    @expectedFailureLinux # this test fails 6/100 dosep runs
+    @expectedFlakeyLinux # this test fails 6/100 dosep runs
     def test_with_dwarf_python(self):
         """Test that we obey thread conditioned breakpoints."""
         self.buildDwarf()
         self.do_thread_specific_break()
 
     def do_thread_specific_break(self):
         """Test that we obey thread conditioned breakpoints."""
         exe = os.path.join(os.getcwd(), "a.out")

test/lang/c/register_variables/TestRegisterVariables.py

class RegisterVariableTestCase(TestBase):
     @expectedFailureDarwin("llvm.org/pr20266")
     def test_with_dsym_and_run_command(self):
         """Test expressions on register values."""
         self.buildDsym()
         self.const_variable()
 
     @dwarf_test
     @expectedFailureClang(None, ['<', '3.5'])
-    @expectedFailureGcc #xfail to get buildbot green, test failed with gcc4.8.2
+    @expectedFlakeyGcc # failed with gcc4.8.2
     def test_with_dwarf_and_run_command(self):
         """Test expressions on register values."""
         self.buildDwarf()
         self.const_variable()
 
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)

test/lang/cpp/this/TestCPPThis.py

def test_with_dsym_and_run_command(self):
         """Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods"""
         self.buildDsym()
         self.static_method_commands()
 
     #rdar://problem/9962849
     @expectedFailureGcc # llvm.org/pr15439 The 'this' pointer isn't available during expression evaluation when stopped in an inlined member function.
     @expectedFailureIcc # ICC doesn't emit correct DWARF inline debug info for inlined member functions
     @dwarf_test
-    @expectedFailureClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6'])#xfail to get buildbot green, test failed with totclang - clang3.7
+    @expectedFlakeyClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6']) # failed with totclang - clang3.7
     def test_with_dwarf_and_run_command(self):
         """Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods"""
         self.buildDwarf()
         self.static_method_commands()
 
     def setUp(self):
         TestBase.setUp(self)
     

test/lldbtest.py

def fn(self):
         if match:
             if not api_levels:
                 return True
             device_api = android_device_api()
             return device_api and (device_api in api_levels)
 
     return expectedFailure(fn, bugnumber)
 
+# if the test passes on the first try, we're done (success)
+# if the test fails once, then passes on the second try, raise an ExpectedFailure
+# if the test fails twice in a row, re-throw the exception from the second test run
+def expectedFlakey(expected_fn, bugnumber=None):
+    def expectedFailure_impl(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            from unittest2 import case
+            self = args[0]
+            try:
+                func(*args, **kwargs)
+            except Exception:
+                if expected_fn(self):
+                    # retry
+                    try:
+                        func(*args, **kwargs)
+                    except Exception:
+                        # oh snap! two failures in a row, record a failure/error
+                        raise
+                    # record the expected failure
+                    raise case._ExpectedFailure(sys.exc_info(), bugnumber)
+                else:
+                    raise
+        return wrapper
+    # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments
+    # return decorator in this case, so it will be used to decorating original method
+    if callable(bugnumber):
+        return expectedFailure_impl(bugnumber)
+    else:
+        return expectedFailure_impl
+
+def expectedFlakeyOS(oslist, bugnumber=None, compilers=None):
+    def fn(self):
+        return (self.getPlatform() in oslist and
+                self.expectedCompiler(compilers))
+    return expectedFlakey(fn, bugnumber)
+
+def expectedFlakeyDarwin(bugnumber=None, compilers=None):
+    # For legacy reasons, we support both "darwin" and "macosx" as OS X triples.
+    return expectedFlakeyOS(getDarwinOSTriples(), bugnumber, compilers)
+
+def expectedFlakeyLinux(bugnumber=None, compilers=None):
+    return expectedFlakeyOS(['linux'], bugnumber, compilers)
+
+def expectedFlakeyFreeBSD(bugnumber=None, compilers=None):
+    return expectedFlakeyOS(['freebsd'], bugnumber, compilers)
+
+def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None):
+    if compiler_version is None:
+        compiler_version=['=', None]
+    def fn(self):
+        return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version)
+    return expectedFlakey(fn, bugnumber)
+
+# @expectedFlakeyClang('bugnumber', ['<=', '3.4'])
+def expectedFlakeyClang(bugnumber=None, compiler_version=None):
+    return expectedFlakeyCompiler('clang', compiler_version, bugnumber)
+
+# @expectedFlakeyGcc('bugnumber', ['<=', '3.4'])
+def expectedFlakeyGcc(bugnumber=None, compiler_version=None):
+    return expectedFlakeyCompiler('gcc', compiler_version, bugnumber)
+
 def skipIfRemote(func):
     """Decorate the item to skip tests if testing remotely."""
     if isinstance(func, type) and issubclass(func, unittest2.TestCase):
         raise Exception("@skipIfRemote can only be used to decorate a test method")
     @wraps(func)
     def wrapper(*args, **kwargs):
         from unittest2 import case
         if lldb.remote_platform:

test/python_api/signals/TestSignalsAPI.py

 """
 Test SBProcess APIs, including ReadMemory(), WriteMemory(), and others.
 """
 
 import os, time
 import unittest2
 import lldb
 from lldbutil import get_stopped_thread, state_type_to_str
 from lldbtest import *
 
 class SignalsAPITestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     @python_api_test
-    @expectedFailureLinux # this test fails 1/100 dosep runs
+    @expectedFlakeyLinux # this test fails 1/100 dosep runs
     def test_ignore_signal(self):
         """Test Python SBUnixSignals.Suppress/Stop/Notify() API."""
         self.buildDefault()
         exe = os.path.join(os.getcwd(), "a.out")
         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
 
         target = self.dbg.CreateTarget(exe)
         self.assertTrue(target, VALID_TARGET)

test/types/TestFloatTypes.py

class FloatTypesTestCase(AbstractBase.GenericTester):
 
     @dwarf_test
     def test_float_type_with_dwarf(self):
         """Test that float-type variables are displayed correctly."""
         self.build_and_run('float.cpp', set(['float']), dsym=False)
 
     @skipUnlessDarwin
     @dsym_test
-    @expectedFailureDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
+    @expectedFlakeyDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110
     def test_double_type_with_dsym(self):
         """Test that double-type variables are displayed correctly."""
         self.build_and_run('double.cpp', set(['double']))
 
     @skipUnlessDarwin
     @dsym_test
     def test_double_type_from_block_with_dsym(self):
         """Test that double-type variables are displayed correctly from a block."""