Index: lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py =================================================================== --- lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py +++ lldb/trunk/test/expression_command/call-function/TestCallStopAndContinue.py @@ -22,14 +22,14 @@ @skipUnlessDarwin @dsym_test - @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX + @expectedFlakeyDarwin("llvm.org/pr20274") def test_with_dsym(self): """Test gathering result from interrupted function call.""" self.buildDsym() self.call_function() @dwarf_test - @expectedFailureDarwin("llvm.org/pr20274") # intermittent failure on MacOSX + @expectedFlakeyDarwin("llvm.org/pr20274") def test_with_dwarf(self): """Test gathering result from interrupted function call.""" self.buildDwarf() Index: lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py =================================================================== --- lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py +++ lldb/trunk/test/expression_command/call-restarts/TestCallThatRestarts.py @@ -29,7 +29,7 @@ @dwarf_test @skipIfFreeBSD # llvm.org/pr19246: intermittent failure - @expectedFailureLinux("llvm.org/pr19246") # intermittent failure + @expectedFailureLinux("llvm.org/pr19246") @skipIfDarwin # llvm.org/pr19246: intermittent failure @skipIfWindows # Test relies on signals, unsupported on Windows def test_with_dwarf(self): Index: lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py =================================================================== --- lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py +++ lldb/trunk/test/expression_command/timeout/TestCallWithTimeout.py @@ -28,7 +28,7 @@ self.call_function() @expectedFailureFreeBSD("llvm.org/pr19605") # fails on buildbot - @expectedFailureLinux("llvm.org/pr20275") # fails intermittently on Linux + @expectedFlakeyLinux("llvm.org/pr20275") @dwarf_test def test_with_dwarf(self): """Test calling std::String member function.""" Index: lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py =================================================================== --- lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py +++ lldb/trunk/test/functionalities/attach_resume/TestAttachResume.py @@ -22,7 +22,7 @@ self.buildDwarf() self.process_attach_continue_interrupt_detach() - @expectedFailureLinux('llvm.org/pr19478') # intermittent ~2/14 runs + @expectedFlakeyLinux('llvm.org/pr19478') # intermittent ~2/14 runs @skipIfRemote def process_attach_continue_interrupt_detach(self): """Test attach/continue/interrupt/detach""" Index: lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py =================================================================== --- lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py +++ lldb/trunk/test/functionalities/avoids-fd-leak/TestFdLeak.py @@ -41,8 +41,8 @@ self.assertTrue(process.GetExitStatus() == 0, "Process returned non-zero status. Were incorrect file descriptors passed?") - @expectedFailure(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd - @expectedFailureLinux # xfail flakey test to get buildbot green + @expectedFlakey(lambda x: sys.version_info >= (2, 7, 8), "bugs.freebsd.org/197376") # python random leaks fd + @expectedFlakeyLinux @skipIfWindows # The check for descriptor leakage needs to be implemented differently here. @skipIfTargetAndroid() # Android have some other file descriptors open by the shell def test_fd_leak_multitarget (self): Index: lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py =================================================================== --- lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py +++ lldb/trunk/test/functionalities/expr-doesnt-deadlock/TestExprDoesntBlock.py @@ -24,7 +24,7 @@ @dwarf_test @expectedFailureFreeBSD('llvm.org/pr17946') - @expectedFailureLinux # failed 1/365 test runs, line 61, thread.IsValid() + @expectedFlakeyLinux # failed 1/365 test runs, line 61, thread.IsValid() def test_with_dwarf_and_run_command(self): """Test that expr will time out and allow other threads to run if it blocks.""" self.buildDwarf() Index: lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py =================================================================== --- lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py +++ lldb/trunk/test/functionalities/stop-hook/TestStopHookMechanism.py @@ -19,7 +19,7 @@ self.stop_hook_firing() @skipIfFreeBSD # llvm.org/pr15037 - @expectedFailureLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux + @expectedFlakeyLinux('llvm.org/pr15037') # stop-hooks sometimes fail to fire on Linux @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows") @dwarf_test def test_with_dwarf(self): Index: lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py =================================================================== --- lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py +++ lldb/trunk/test/functionalities/stop-hook/multiple_threads/TestStopHookMultipleThreads.py @@ -21,7 +21,7 @@ @dwarf_test @expectedFailureFreeBSD("llvm.org/pr15037") - @expectedFailureLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux + @expectedFlakeyLinux("llvm.org/pr15037") # stop hooks sometimes fail to fire on Linux @expectedFailureWindows("llvm.org/pr22274: need a pexpect replacement for windows") @skipIfFreeBSD # llvm.org/pr22784 def test_stop_hook_multiple_threads_with_dwarf(self): Index: lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py =================================================================== --- lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py +++ lldb/trunk/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py @@ -22,6 +22,8 @@ @skipIfFreeBSD # Hangs. May be the same as Linux issue llvm.org/pr16229 but # not yet investigated. Revisit once required functionality # is implemented for FreeBSD. + @skipIfLinux # Occasionally hangs on the build bot, expectedFailureLinux + @dwarf_test def test_create_after_attach_with_dwarf_and_popen(self): """Test thread creation after process attach.""" @@ -32,6 +34,7 @@ # for FreeBSD. @dwarf_test @skipIfRemote + @expectedFlakeyLinux("llvm.org/pr16229") # 1/100 dosep, build 3546, clang-3.5 x84_64 def test_create_after_attach_with_dwarf_and_fork(self): """Test thread creation after process attach.""" self.buildDwarf(dictionary=self.getBuildFlags(use_cpp11=False)) Index: lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py =================================================================== --- lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py +++ lldb/trunk/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py @@ -23,7 +23,7 @@ @expectedFailureFreeBSD('llvm.org/pr18522') # hits break in another thread in testrun @python_api_test @dwarf_test - @expectedFailureLinux # this test fails 6/100 dosep runs + @expectedFlakeyLinux # this test fails 6/100 dosep runs def test_with_dwarf_python(self): """Test that we obey thread conditioned breakpoints.""" self.buildDwarf() Index: lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py =================================================================== --- lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py +++ lldb/trunk/test/lang/c/register_variables/TestRegisterVariables.py @@ -19,7 +19,7 @@ @dwarf_test @expectedFailureClang(None, ['<', '3.5']) - @expectedFailureGcc #xfail to get buildbot green, test failed with gcc4.8.2 + @expectedFlakeyGcc # failed with gcc4.8.2 def test_with_dwarf_and_run_command(self): """Test expressions on register values.""" self.buildDwarf() Index: lldb/trunk/test/lang/cpp/this/TestCPPThis.py =================================================================== --- lldb/trunk/test/lang/cpp/this/TestCPPThis.py +++ lldb/trunk/test/lang/cpp/this/TestCPPThis.py @@ -22,7 +22,7 @@ @expectedFailureGcc # llvm.org/pr15439 The 'this' pointer isn't available during expression evaluation when stopped in an inlined member function. @expectedFailureIcc # ICC doesn't emit correct DWARF inline debug info for inlined member functions @dwarf_test - @expectedFailureClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6'])#xfail to get buildbot green, test failed with totclang - clang3.7 + @expectedFlakeyClang(bugnumber='llvm.org/pr23012', compiler_version=['>=','3.6']) # failed with totclang - clang3.7 def test_with_dwarf_and_run_command(self): """Test that the appropriate member variables are available when stopped in C++ static, inline, and const methods""" self.buildDwarf() Index: lldb/trunk/test/lldbtest.py =================================================================== --- lldb/trunk/test/lldbtest.py +++ lldb/trunk/test/lldbtest.py @@ -712,6 +712,68 @@ return expectedFailure(fn, bugnumber) +# if the test passes on the first try, we're done (success) +# if the test fails once, then passes on the second try, raise an ExpectedFailure +# if the test fails twice in a row, re-throw the exception from the second test run +def expectedFlakey(expected_fn, bugnumber=None): + def expectedFailure_impl(func): + @wraps(func) + def wrapper(*args, **kwargs): + from unittest2 import case + self = args[0] + try: + func(*args, **kwargs) + except Exception: + if expected_fn(self): + # retry + try: + func(*args, **kwargs) + except Exception: + # oh snap! two failures in a row, record a failure/error + raise + # record the expected failure + raise case._ExpectedFailure(sys.exc_info(), bugnumber) + else: + raise + return wrapper + # if bugnumber is not-callable(incluing None), that means decorator function is called with optional arguments + # return decorator in this case, so it will be used to decorating original method + if callable(bugnumber): + return expectedFailure_impl(bugnumber) + else: + return expectedFailure_impl + +def expectedFlakeyOS(oslist, bugnumber=None, compilers=None): + def fn(self): + return (self.getPlatform() in oslist and + self.expectedCompiler(compilers)) + return expectedFlakey(fn, bugnumber) + +def expectedFlakeyDarwin(bugnumber=None, compilers=None): + # For legacy reasons, we support both "darwin" and "macosx" as OS X triples. + return expectedFlakeyOS(getDarwinOSTriples(), bugnumber, compilers) + +def expectedFlakeyLinux(bugnumber=None, compilers=None): + return expectedFlakeyOS(['linux'], bugnumber, compilers) + +def expectedFlakeyFreeBSD(bugnumber=None, compilers=None): + return expectedFlakeyOS(['freebsd'], bugnumber, compilers) + +def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None): + if compiler_version is None: + compiler_version=['=', None] + def fn(self): + return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version) + return expectedFlakey(fn, bugnumber) + +# @expectedFlakeyClang('bugnumber', ['<=', '3.4']) +def expectedFlakeyClang(bugnumber=None, compiler_version=None): + return expectedFlakeyCompiler('clang', compiler_version, bugnumber) + +# @expectedFlakeyGcc('bugnumber', ['<=', '3.4']) +def expectedFlakeyGcc(bugnumber=None, compiler_version=None): + return expectedFlakeyCompiler('gcc', compiler_version, bugnumber) + def skipIfRemote(func): """Decorate the item to skip tests if testing remotely.""" if isinstance(func, type) and issubclass(func, unittest2.TestCase): Index: lldb/trunk/test/python_api/signals/TestSignalsAPI.py =================================================================== --- lldb/trunk/test/python_api/signals/TestSignalsAPI.py +++ lldb/trunk/test/python_api/signals/TestSignalsAPI.py @@ -12,7 +12,7 @@ mydir = TestBase.compute_mydir(__file__) @python_api_test - @expectedFailureLinux # this test fails 1/100 dosep runs + @expectedFlakeyLinux # this test fails 1/100 dosep runs def test_ignore_signal(self): """Test Python SBUnixSignals.Suppress/Stop/Notify() API.""" self.buildDefault() Index: lldb/trunk/test/types/TestFloatTypes.py =================================================================== --- lldb/trunk/test/types/TestFloatTypes.py +++ lldb/trunk/test/types/TestFloatTypes.py @@ -38,7 +38,7 @@ @skipUnlessDarwin @dsym_test - @expectedFailureDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110 + @expectedFlakeyDarwin # failed 1/140 runs 'frame variable --show-types a_union_nonzero_ref.u.a' matches the output (from compiled code): 11001110 def test_double_type_with_dsym(self): """Test that double-type variables are displayed correctly.""" self.build_and_run('double.cpp', set(['double']))