Index: test/CMakeLists.txt =================================================================== --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -118,6 +118,11 @@ ) endif() +option(LLVM_XFAILS_REQUIRE_PRS "Require PRs for all tests with XFAIL annotations" Off) +if(LLVM_XFAILS_REQUIRE_PRS) + list(APPEND LLVM_LIT_ARGS "--xfail-requires-pr") +endif() + add_custom_target(llvm-test-depends DEPENDS ${LLVM_TEST_DEPENDS}) set_target_properties(llvm-test-depends PROPERTIES FOLDER "Tests") Index: utils/lit/lit/LitConfig.py =================================================================== --- utils/lit/lit/LitConfig.py +++ utils/lit/lit/LitConfig.py @@ -24,7 +24,8 @@ noExecute, debug, isWindows, params, config_prefix = None, maxIndividualTestTime = 0, - maxFailures = None): + maxFailures = None, + xFailRequirePR = False): # The name of the test runner. self.progname = progname # The items to add to the PATH environment variable. @@ -62,6 +63,7 @@ self.maxIndividualTestTime = maxIndividualTestTime self.maxFailures = maxFailures + self.xFailRequirePR = xFailRequirePR @property def maxIndividualTestTime(self): Index: utils/lit/lit/Test.py =================================================================== --- utils/lit/lit/Test.py +++ utils/lit/lit/Test.py @@ -183,7 +183,7 @@ # A list of conditions under which this test is expected to fail. These # can optionally be provided by test format handlers, and will be # honored when the test result is supplied. - self.xfails = [] + self.xfails = {} # The test result, once complete. self.result = None @@ -227,17 +227,17 @@ """ # Check if any of the xfails match an available feature or the target. - for item in self.xfails: + for feature, pr in self.xfails.iteritems(): # If this is the wildcard, it always fails. - if item == '*': + if feature == '*': return True # If this is an exact match for one of the features, it fails. - if item in self.config.available_features: + if feature in self.config.available_features: return True # If this is a part of the target triple, it fails. - if item and item in self.suite.config.target_triple: + if feature and feature in self.suite.config.target_triple: return True return False Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -672,7 +672,7 @@ # convert to list before returning. return list(map(processLine, script)) -def parseIntegratedTestScript(test, require_script=True): +def parseIntegratedTestScript(test, require_script=True, xFailRequirePR=False): """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' and 'UNSUPPORTED' information. If 'require_script' is False an empty script @@ -708,7 +708,15 @@ else: script.append(ln) elif command_type == 'XFAIL': - test.xfails.extend([s.strip() for s in ln.split(',')]) + xfail_prs = [s.strip() for s in ln.split(',') if s.startswith("PR")] + if len(xfail_prs) > 1: + raise ValueError("XFAIL command cannot have more than one PR") + xfail_pr = None + if(len(xfail_prs) > 0): + xfail_pr = xfail_prs[0] + xfail_features = [s.strip() for s in ln.split(',') if not s.startswith("PR")] + for feature in xfail_features: + test.xfails[feature] = xfail_pr elif command_type == 'REQUIRES': requires.extend([s.strip() for s in ln.split(',')]) elif command_type == 'REQUIRES-ANY': @@ -768,6 +776,11 @@ return lit.Test.Result(Test.UNSUPPORTED, "Test requires one of the limit_to_features features %s" % msg) + if xFailRequirePR: + for feature, pr in test.xfails.iteritems(): + if pr is None: + return lit.Test.Result(Test.FAIL, "XFAIL command missing PR") + return script def _runShTest(test, litConfig, useExternalSh, script, tmpBase): @@ -813,7 +826,8 @@ if test.config.unsupported: return (Test.UNSUPPORTED, 'Test is unsupported') - script = parseIntegratedTestScript(test) + script = parseIntegratedTestScript(test, + xFailRequirePR=litConfig.xFailRequirePR) if isinstance(script, lit.Test.Result): return script if litConfig.noExecute: Index: utils/lit/lit/main.py =================================================================== --- utils/lit/lit/main.py +++ utils/lit/lit/main.py @@ -273,6 +273,9 @@ group.add_option("", "--use-threads", dest="useProcesses", help="Run tests in parallel with threads (not processes)", action="store_false", default=True) + group.add_option("", "--xfail-requires-pr", dest="xFailRequirePR", + help="Require PRs for XFAIL tests", + action="store_true", default=False) parser.add_option_group(group) (opts, args) = parser.parse_args() @@ -331,7 +334,8 @@ params = userParams, config_prefix = opts.configPrefix, maxIndividualTestTime = maxIndividualTestTime, - maxFailures = opts.maxFailures) + maxFailures = opts.maxFailures, + xFailRequirePR = opts.xFailRequirePR) # Perform test discovery. run = lit.run.Run(litConfig,