Index: test/CMakeLists.txt =================================================================== --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -118,6 +118,11 @@ ) endif() +option(LLVM_XFAILS_REQUIRE_PRS "Require PRs for all tests with XFAIL annotations" Off) +if(LLVM_XFAILS_REQUIRE_PRS) + list(APPEND LLVM_LIT_ARGS "--xfail-requires-pr") +endif() + add_custom_target(llvm-test-depends DEPENDS ${LLVM_TEST_DEPENDS}) set_target_properties(llvm-test-depends PROPERTIES FOLDER "Tests") Index: test/ExecutionEngine/MCJIT/stubs-sm-pic.ll =================================================================== --- test/ExecutionEngine/MCJIT/stubs-sm-pic.ll +++ test/ExecutionEngine/MCJIT/stubs-sm-pic.ll @@ -1,5 +1,5 @@ ; RUN: %lli -disable-lazy-compilation=false -relocation-model=pic -code-model=small %s -; XFAIL: mips-, mipsel-, i686, i386, aarch64, arm +; XFAIL: mips-, mipsel-, i686, i386, aarch64, arm, http://llvm.org/PR12345 define i32 @main() nounwind { entry: Index: utils/lit/lit/LitConfig.py =================================================================== --- utils/lit/lit/LitConfig.py +++ utils/lit/lit/LitConfig.py @@ -24,7 +24,9 @@ noExecute, debug, isWindows, params, config_prefix = None, maxIndividualTestTime = 0, - maxFailures = None): + maxFailures = None, + xFailRequirePR = False, + bugTrackerPrefixes = []): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -62,6 +64,8 @@ self.maxIndividualTestTime = maxIndividualTestTime self.maxFailures = maxFailures + self.xFailRequirePR = xFailRequirePR + self.bugTrackerPrefixes = bugTrackerPrefixes @property def maxIndividualTestTime(self): Index: utils/lit/lit/Test.py =================================================================== --- utils/lit/lit/Test.py +++ utils/lit/lit/Test.py @@ -34,6 +34,7 @@ UNRESOLVED = ResultCode('UNRESOLVED', True) UNSUPPORTED = ResultCode('UNSUPPORTED', False) TIMEOUT = ResultCode('TIMEOUT', True) +UNXFAIL = ResultCode('UNXFAIL', False) # Test metric values. @@ -183,7 +184,7 @@ # A list of conditions under which this test is expected to fail. These # can optionally be provided by test format handlers, and will be # honored when the test result is supplied. - self.xfails = [] + self.xfails = {} # The test result, once complete. self.result = None @@ -201,6 +202,8 @@ self.result.code = XPASS elif self.result.code == FAIL: self.result.code = XFAIL + elif self.result.code == UNXFAIL: + self.result.code = FAIL def getFullName(self): return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) @@ -227,17 +230,17 @@ """ # Check if any of the xfails match an available feature or the target. - for item in self.xfails: + for feature, pr in self.xfails.iteritems(): # If this is the wildcard, it always fails. - if item == '*': + if feature == '*': return True # If this is an exact match for one of the features, it fails. - if item in self.config.available_features: + if feature in self.config.available_features: return True # If this is a part of the target triple, it fails. - if item and item in self.suite.config.target_triple: + if feature and feature in self.suite.config.target_triple: return True return False Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -687,6 +687,8 @@ unsupported = [] keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'REQUIRES-ANY:', 'UNSUPPORTED:', 'END.'] + + bugTrackerRegex = re.compile('^' + '|^'.join(test.config.bugTrackerPrefixes)) for line_number, command_type, ln in \ parseIntegratedTestScriptCommands(sourcepath, keywords): if command_type == 'RUN': @@ -708,7 +710,16 @@ else: script.append(ln) elif command_type == 'XFAIL': - test.xfails.extend([s.strip() for s in ln.split(',')]) + xfail_prs = [s.strip() for s in ln.split(',') if bugTrackerRegex.match(s.strip())] + if len(xfail_prs) > 1: + raise ValueError("XFAIL command cannot have more than one PR, " + "use multiple xfail commands.") + xfail_pr = None + if(len(xfail_prs) > 0): + xfail_pr = xfail_prs[0] + xfail_features = [s.strip() for s in ln.split(',') if not bugTrackerRegex.match(s.strip())] + for feature in xfail_features: + test.xfails[feature] = xfail_pr elif command_type == 'REQUIRES': requires.extend([s.strip() for s in ln.split(',')]) elif command_type == 'REQUIRES-ANY': @@ -768,6 +779,11 @@ return lit.Test.Result(Test.UNSUPPORTED, "Test requires one of the limit_to_features features %s" % msg) + if test.config.xFailRequirePR: + for feature, pr in test.xfails.iteritems(): + if pr is None: + return lit.Test.Result(Test.UNXFAIL, "XFAIL command missing PR") + return script def _runShTest(test, litConfig, useExternalSh, script, tmpBase): Index: utils/lit/lit/TestingConfig.py =================================================================== --- utils/lit/lit/TestingConfig.py +++ utils/lit/lit/TestingConfig.py @@ -131,6 +131,11 @@ # Whether the suite should be tested early in a given run. self.is_early = bool(is_early) + # Default this to on at the test suite level, but globally it is off. + # This allows test suites to opt out on a per-suite basis. + self.xFailRequirePR = True + self.bugTrackerPrefixes = [] + def finish(self, litConfig): """finish() - Finish this config object, after loading is complete.""" @@ -147,6 +152,8 @@ # files. Should we distinguish them? self.test_source_root = str(self.test_source_root) self.excludes = set(self.excludes) + self.xFailRequirePR = self.xFailRequirePR and litConfig.xFailRequirePR + self.bugTrackerPrefixes.extend(litConfig.bugTrackerPrefixes) @property def root(self): Index: utils/lit/lit/main.py =================================================================== --- utils/lit/lit/main.py +++ utils/lit/lit/main.py @@ -273,6 +273,14 @@ group.add_option("", "--use-threads", dest="useProcesses", help="Run tests in parallel with threads (not processes)", action="store_false", default=True) + group.add_option("", "--xfail-requires-pr", dest="xFailRequirePR", + help="Require PRs for XFAIL tests", + action="store_true", default=False) + parser.add_option("", "--bug-tracker-prefix", dest="bugTrackerPrefixes", + metavar="", + help=("Prefixes for specifying bugs in XFAIL directives " + "(defaults to include http://llvm.org/PR)"), + type=str, action="append", default=['http://llvm.org/PR']) parser.add_option_group(group) (opts, args) = parser.parse_args() @@ -331,7 +339,9 @@ params = userParams, config_prefix = opts.configPrefix, maxIndividualTestTime = maxIndividualTestTime, - maxFailures = opts.maxFailures) + maxFailures = opts.maxFailures, + xFailRequirePR = opts.xFailRequirePR, + bugTrackerPrefixes = opts.bugTrackerPrefixes) # Perform test discovery. run = lit.run.Run(litConfig, Index: utils/lit/tests/Inputs/xfail-require-pr/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/lit.cfg @@ -0,0 +1,7 @@ +import lit.formats +config.name = 'xfail-requires-pr' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.target_triple = 'x86_64-unknown-unknown' Index: utils/lit/tests/Inputs/xfail-require-pr/local-override-disable/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/local-override-disable/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +config.name = 'xfail-requires-pr-disabled' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.target_triple = 'x86_64-unknown-unknown' +config.xFailRequirePR = False Index: utils/lit/tests/Inputs/xfail-require-pr/local-override-disable/xfail-suite-disabled.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/local-override-disable/xfail-suite-disabled.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: * Index: utils/lit/tests/Inputs/xfail-require-pr/local-override-prefix/lit.cfg =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/local-override-prefix/lit.cfg @@ -0,0 +1,8 @@ +import lit.formats +config.name = 'xfail-requires-pr-custom' +config.suffixes = ['.txt'] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +config.target_triple = 'x86_64-unknown-unknown' +config.bugTrackerPrefixes = ['http://bar.org/'] Index: utils/lit/tests/Inputs/xfail-require-pr/local-override-prefix/xfail-suite-custom-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/local-override-prefix/xfail-suite-custom-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: *, http://bar.org/PR123 Index: utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-custom-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-custom-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: *, http://foo.org/PR123 Index: utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/xfail-containing-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: *, http://llvm.org/PR123 Index: utils/lit/tests/Inputs/xfail-require-pr/xfail-missing-pr.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/xfail-require-pr/xfail-missing-pr.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: * Index: utils/lit/tests/xfail-require-pr.py =================================================================== --- /dev/null +++ utils/lit/tests/xfail-require-pr.py @@ -0,0 +1,26 @@ +# RUN: %{lit} -j 1 -v %{inputs}/xfail-require-pr | FileCheck --check-prefix=DEFAULT %s +# DEFAULT: XFAIL: xfail-requires-pr :: xfail-containing-custom-pr.txt (1 of 5) +# DEFAULT: XFAIL: xfail-requires-pr :: xfail-containing-pr.txt (2 of 5) +# DEFAULT: XFAIL: xfail-requires-pr :: xfail-missing-pr.txt (3 of 5) +# DEFAULT: XFAIL: xfail-requires-pr-custom :: xfail-suite-custom-pr.txt (4 of 5) +# DEFAULT: XFAIL: xfail-requires-pr-disabled :: xfail-suite-disabled.txt (5 of 5) +# DEFAULT: Expected Failures : 5 + + +# RUN: not %{lit} -j 1 -v %{inputs}/xfail-require-pr --xfail-requires-pr | FileCheck --check-prefix=REQUIRE_PR %s +# REQUIRE_PR: FAIL: xfail-requires-pr :: xfail-containing-custom-pr.txt (1 of 5) +# REQUIRE_PR: XFAIL: xfail-requires-pr :: xfail-containing-pr.txt (2 of 5) +# REQUIRE_PR: FAIL: xfail-requires-pr :: xfail-missing-pr.txt (3 of 5) +# REQUIRE_PR: XFAIL: xfail-requires-pr-custom :: xfail-suite-custom-pr.txt (4 of 5) +# REQUIRE_PR: XFAIL: xfail-requires-pr-disabled :: xfail-suite-disabled.txt (5 of 5) +# REQUIRE_PR: Expected Failures : 3 +# REQUIRE_PR: Unexpected Failures: 2 + +# RUN: not %{lit} -j 1 -v %{inputs}/xfail-require-pr --xfail-requires-pr --bug-tracker-prefix http://foo.org/ | FileCheck --check-prefix=CUSTOM_PR %s +# CUSTOM_PR: XFAIL: xfail-requires-pr :: xfail-containing-custom-pr.txt (1 of 5) +# CUSTOM_PR: XFAIL: xfail-requires-pr :: xfail-containing-pr.txt (2 of 5) +# CUSTOM_PR: FAIL: xfail-requires-pr :: xfail-missing-pr.txt (3 of 5) +# CUSTOM_PR: XFAIL: xfail-requires-pr-custom :: xfail-suite-custom-pr.txt (4 of 5) +# CUSTOM_PR: XFAIL: xfail-requires-pr-disabled :: xfail-suite-disabled.txt (5 of 5) +# CUSTOM_PR: Expected Failures : 4 +# CUSTOM_PR: Unexpected Failures: 1