Index: test/CMakeLists.txt =================================================================== --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -118,6 +118,11 @@ ) endif() +option(LLVM_XFAILS_REQUIRE_PRS "Require PRs for all tests with XFAIL annotations" Off) +if(LLVM_XFAILS_REQUIRE_PRS) + list(APPEND LLVM_LIT_ARGS "--xfail-requires-pr") +endif() + add_custom_target(llvm-test-depends DEPENDS ${LLVM_TEST_DEPENDS}) set_target_properties(llvm-test-depends PROPERTIES FOLDER "Tests") Index: test/ExecutionEngine/MCJIT/stubs-sm-pic.ll =================================================================== --- test/ExecutionEngine/MCJIT/stubs-sm-pic.ll +++ test/ExecutionEngine/MCJIT/stubs-sm-pic.ll @@ -1,5 +1,5 @@ ; RUN: %lli -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s -; XFAIL: mips-, mipsel-, i686, i386, aarch64, arm +; XFAIL: mips-, mipsel-, i686, i386, aarch64, arm, http://llvm.org/PR12345 define i32 @main() nounwind { entry: Index: utils/lit/lit/LitConfig.py =================================================================== --- utils/lit/lit/LitConfig.py +++ utils/lit/lit/LitConfig.py @@ -24,7 +24,9 @@ noExecute, debug, isWindows, params, config_prefix = None, maxIndividualTestTime = 0, - maxFailures = None): + maxFailures = None, + xFailRequirePR = False, + bugTrackerRegex = None): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -62,6 +64,8 @@ self.maxIndividualTestTime = maxIndividualTestTime self.maxFailures = maxFailures + self.xFailRequirePR = xFailRequirePR + self.bugTrackerRegex = bugTrackerRegex @property def maxIndividualTestTime(self): Index: utils/lit/lit/Test.py =================================================================== --- utils/lit/lit/Test.py +++ utils/lit/lit/Test.py @@ -34,6 +34,7 @@ UNRESOLVED = ResultCode('UNRESOLVED', True) UNSUPPORTED = ResultCode('UNSUPPORTED', False) TIMEOUT = ResultCode('TIMEOUT', True) +UNXFAIL = ResultCode('UNXFAIL', False) # Test metric values. @@ -183,7 +184,7 @@ # A list of conditions under which this test is expected to fail. These # can optionally be provided by test format handlers, and will be # honored when the test result is supplied. - self.xfails = [] + self.xfails = {} # The test result, once complete. self.result = None @@ -201,6 +202,8 @@ self.result.code = XPASS elif self.result.code == FAIL: self.result.code = XFAIL + elif self.result.code == UNXFAIL: + self.result.code = FAIL def getFullName(self): return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) @@ -227,17 +230,17 @@ """ # Check if any of the xfails match an available feature or the target. - for item in self.xfails: + for feature, pr in self.xfails.iteritems(): # If this is the wildcard, it always fails. - if item == '*': + if feature == '*': return True # If this is an exact match for one of the features, it fails. - if item in self.config.available_features: + if feature in self.config.available_features: return True # If this is a part of the target triple, it fails. - if item and item in self.suite.config.target_triple: + if feature and feature in self.suite.config.target_triple: return True return False Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -672,7 +672,7 @@ # convert to list before returning. return list(map(processLine, script)) -def parseIntegratedTestScript(test, require_script=True): +def parseIntegratedTestScript(test, require_script=True, xFailRequirePR=False, bugTrackerRegex=None): """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' and 'UNSUPPORTED' information. If 'require_script' is False an empty script @@ -708,7 +708,16 @@ else: script.append(ln) elif command_type == 'XFAIL': - test.xfails.extend([s.strip() for s in ln.split(',')]) + xfail_prs = [s.strip() for s in ln.split(',') if bugTrackerRegex.match(s.strip())] + if len(xfail_prs) > 1: + raise ValueError("XFAIL command cannot have more than one PR, " + "use multiple xfail commands.") + xfail_pr = None + if(len(xfail_prs) > 0): + xfail_pr = xfail_prs[0] + xfail_features = [s.strip() for s in ln.split(',') if not bugTrackerRegex.match(s.strip())] + for feature in xfail_features: + test.xfails[feature] = xfail_pr elif command_type == 'REQUIRES': requires.extend([s.strip() for s in ln.split(',')]) elif command_type == 'REQUIRES-ANY': @@ -768,6 +777,11 @@ return lit.Test.Result(Test.UNSUPPORTED, "Test requires one of the limit_to_features features %s" % msg) + if xFailRequirePR: + for feature, pr in test.xfails.iteritems(): + if pr is None: + return lit.Test.Result(Test.UNXFAIL, "XFAIL command missing PR") + return script def _runShTest(test, litConfig, useExternalSh, script, tmpBase): @@ -813,7 +827,9 @@ if test.config.unsupported: return (Test.UNSUPPORTED, 'Test is unsupported') - script = parseIntegratedTestScript(test) + script = parseIntegratedTestScript(test, + xFailRequirePR=litConfig.xFailRequirePR, + bugTrackerRegex=litConfig.bugTrackerRegex) if isinstance(script, lit.Test.Result): return script if litConfig.noExecute: Index: utils/lit/lit/main.py =================================================================== --- utils/lit/lit/main.py +++ utils/lit/lit/main.py @@ -273,6 +273,14 @@ group.add_option("", "--use-threads", dest="useProcesses", help="Run tests in parallel with threads (not processes)", action="store_false", default=True) + group.add_option("", "--xfail-requires-pr", dest="xFailRequirePR", + help="Require PRs for XFAIL tests", + action="store_true", default=False) + parser.add_option("", "--bug-tracker-prefix", dest="bugTrackerPrefixes", + metavar="", + help=("Prefixes for specifying bugs in XFAIL directives " + "(defaults to include http://llvm.org/PR)"), + type=str, action="append", default=['http://llvm.org/PR']) parser.add_option_group(group) (opts, args) = parser.parse_args() @@ -316,6 +324,7 @@ maxIndividualTestTime = 0 isWindows = platform.system() == 'Windows' + bugTrackerRegex = re.compile('^' + '|^'.join(opts.bugTrackerPrefixes)) # Create the global config object. litConfig = lit.LitConfig.LitConfig( @@ -331,7 +340,9 @@ params = userParams, config_prefix = opts.configPrefix, maxIndividualTestTime = maxIndividualTestTime, - maxFailures = opts.maxFailures) + maxFailures = opts.maxFailures, + xFailRequirePR = opts.xFailRequirePR, + bugTrackerRegex = bugTrackerRegex) # Perform test discovery. run = lit.run.Run(litConfig, Index: utils/lit/tests/Inputs/xfail-require-pr/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/lit.cfg @@ -0,0 +1,7 @@ +import lit.formats +config.name = 'xfail-requires-pr' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.target_triple = 'x86_64-unknown-unknown' Index: utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-custom-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-custom-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: *, http://foo.org/PR123 Index: utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: *, http://llvm.org/PR123 Index: utils/lit/tests/Inputs/xfail-require-pr/xfail-missing-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/xfail-missing-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: * Index: utils/lit/tests/xfail-require-pr.py =================================================================== --- /dev/null +++ utils/lit/tests/xfail-require-pr.py @@ -0,0 +1,20 @@ +# RUN: %{lit} -j 1 -v %{inputs}/xfail-require-pr | FileCheck --check-prefix=DEFAULT %s +# DEFAULT: XFAIL: xfail-requires-pr :: xfail-containing-custom-pr.txt (1 of 3) +# DEFAULT: XFAIL: xfail-requires-pr :: xfail-containing-pr.txt (2 of 3) +# DEFAULT: XFAIL: xfail-requires-pr :: xfail-missing-pr.txt (3 of 3) +# DEFAULT: Expected Failures : 3 + + +# RUN: not %{lit} -j 1 -v %{inputs}/xfail-require-pr --xfail-requires-pr | FileCheck --check-prefix=REQUIRE_PR %s +# REQUIRE_PR: FAIL: xfail-requires-pr :: xfail-containing-custom-pr.txt (1 of 3) +# REQUIRE_PR: XFAIL: xfail-requires-pr :: xfail-containing-pr.txt (2 of 3) +# REQUIRE_PR: FAIL: xfail-requires-pr :: xfail-missing-pr.txt (3 of 3) +# REQUIRE_PR: Expected Failures : 1 +# REQUIRE_PR: Unexpected Failures: 2 + +# RUN: not %{lit} -j 1 -v %{inputs}/xfail-require-pr --xfail-requires-pr --bug-tracker-prefix http://foo.org/ | FileCheck --check-prefix=CUSTOM_PR %s +# CUSTOM_PR: XFAIL: xfail-requires-pr :: xfail-containing-custom-pr.txt (1 of 3) +# CUSTOM_PR: XFAIL: xfail-requires-pr :: xfail-containing-pr.txt (2 of 3) +# CUSTOM_PR: FAIL: xfail-requires-pr :: xfail-missing-pr.txt (3 of 3) +# CUSTOM_PR: Expected Failures : 2 +# CUSTOM_PR: Unexpected Failures: 1