diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -220,6 +220,11 @@
# triple parts. All of them must be False for the test to run.
self.unsupported = []
+ # An optional number of times to allow the test to fail before it
+ # succeeds. The test is basically run until it succeeds, or until
+ # it has failed the number of times specified here.
+ self.allowed_failures = getattr(config, 'test_retry_attempts', 0)
+
# The test result, once complete.
self.result = None
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1182,13 +1182,15 @@
LIST: A keyword taking a comma-separated list of values.
BOOLEAN_EXPR: A keyword taking a comma-separated list of
boolean expressions. Ex 'XFAIL:'
+ INTEGER: A keyword taking a single integer. Ex 'ALLOW_FAILURES:'
CUSTOM: A keyword with custom parsing semantics.
"""
TAG = 0
COMMAND = 1
LIST = 2
BOOLEAN_EXPR = 3
- CUSTOM = 4
+ INTEGER = 4
+ CUSTOM = 5
@staticmethod
def allowedKeywordSuffixes(value):
@@ -1196,6 +1198,7 @@
ParserKind.COMMAND: [':'],
ParserKind.LIST: [':'],
ParserKind.BOOLEAN_EXPR: [':'],
+ ParserKind.INTEGER: [':'],
ParserKind.CUSTOM: [':', '.']
} [value]
@@ -1205,6 +1208,7 @@
ParserKind.COMMAND: 'COMMAND',
ParserKind.LIST: 'LIST',
ParserKind.BOOLEAN_EXPR: 'BOOLEAN_EXPR',
+ ParserKind.INTEGER: 'INTEGER',
ParserKind.CUSTOM: 'CUSTOM'
} [value]
@@ -1247,6 +1251,8 @@
self.parser = self._handleList
elif kind == ParserKind.BOOLEAN_EXPR:
self.parser = self._handleBooleanExpr
+ elif kind == ParserKind.INTEGER:
+ self.parser = self._handleSingleInteger
elif kind == ParserKind.TAG:
self.parser = self._handleTag
elif kind == ParserKind.CUSTOM:
@@ -1311,6 +1317,18 @@
output.extend([s.strip() for s in line.split(',')])
return output
+ @staticmethod
+ def _handleSingleInteger(line_number, line, output):
+ """A parser for INTEGER type keywords"""
+ if output is None:
+ output = []
+ try:
+ n = int(line)
+ except ValueError:
+ raise ValueError("INTEGER parser requires the input to be an integer (got {})".format(line))
+ output.append(n)
+ return output
+
@staticmethod
def _handleBooleanExpr(line_number, line, output):
"""A parser for BOOLEAN_EXPR type keywords"""
@@ -1331,8 +1349,8 @@
def parseIntegratedTestScript(test, additional_parsers=[],
require_script=True):
"""parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
- script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
- and 'UNSUPPORTED' information.
+ script and extract the lines to 'RUN' as well as 'XFAIL', 'REQUIRES',
+ 'UNSUPPORTED' and 'ALLOW_FAILURES' information.
If additional parsers are specified then the test is also scanned for the
keywords they specify and all matches are passed to the custom parser.
@@ -1353,6 +1371,7 @@
initial_value=test.requires),
IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
initial_value=test.unsupported),
+ IntegratedTestKeywordParser('ALLOW_FAILURES:', ParserKind.INTEGER),
IntegratedTestKeywordParser('END.', ParserKind.TAG)
]
keyword_parsers = {p.keyword: p for p in builtin_parsers}
@@ -1412,6 +1431,14 @@
"Test does not support the following features "
"and/or targets: %s" % msg)
+ # Handle ALLOW_FAILURES:
+ allowed_failures = keyword_parsers['ALLOW_FAILURES:'].getValue()
+ if allowed_failures:
+ if len(allowed_failures) > 1:
+ return lit.Test.Result(Test.UNRESOLVED,
+ "Test has more than one ALLOW_FAILURES lines")
+ test.allowed_failures = allowed_failures[0]
+
# Enforce limit_to_features.
if not test.isWithinFeatureLimits():
msg = ', '.join(test.config.limit_to_features)
@@ -1477,10 +1504,8 @@
normalize_slashes=useExternalSh)
script = applySubstitutions(script, substitutions)
- # Re-run failed tests up to test_retry_attempts times.
- attempts = 1
- if hasattr(test.config, 'test_retry_attempts'):
- attempts += test.config.test_retry_attempts
+ # Re-run failed tests up to test.allowed_failures times.
+ attempts = test.allowed_failures + 1
for i in range(attempts):
res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
if res.code != Test.FAIL:
diff --git a/llvm/utils/lit/tests/Inputs/allowed-failures/does-not-succeed-within-limit.py b/llvm/utils/lit/tests/Inputs/allowed-failures/does-not-succeed-within-limit.py
new file mode 100644
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/allowed-failures/does-not-succeed-within-limit.py
@@ -0,0 +1,4 @@
+# ALLOW_FAILURES: 3
+
+# This test does not succeed within the allowed retry limit
+# RUN: false
diff --git a/llvm/utils/lit/tests/Inputs/allowed-failures/lit.cfg b/llvm/utils/lit/tests/Inputs/allowed-failures/lit.cfg
new file mode 100644
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/allowed-failures/lit.cfg
@@ -0,0 +1,9 @@
+import lit.formats
+config.name = 'allowed-failures'
+config.suffixes = ['.py']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+
+config.substitutions.append(('%python', lit_config.params.get('python', '')))
+config.substitutions.append(('%counter', lit_config.params.get('counter', '')))
diff --git a/llvm/utils/lit/tests/Inputs/allowed-failures/more-than-one-allow-failures-lines.py b/llvm/utils/lit/tests/Inputs/allowed-failures/more-than-one-allow-failures-lines.py
new file mode 100644
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/allowed-failures/more-than-one-allow-failures-lines.py
@@ -0,0 +1,6 @@
+# ALLOW_FAILURES: 3
+# ALLOW_FAILURES: 5
+
+# This test should be UNRESOLVED since it has more than one ALLOW_FAILURES:
+# lines, and that is not allowed.
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/allowed-failures/not-a-valid-integer.py b/llvm/utils/lit/tests/Inputs/allowed-failures/not-a-valid-integer.py
new file mode 100644
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/allowed-failures/not-a-valid-integer.py
@@ -0,0 +1,5 @@
+# ALLOW_FAILURES: not-an-integer
+
+# This test does not provide a valid integer to the ALLOW_FAILURES: keyword.
+# It should be unresolved.
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/allowed-failures/succeeds-within-limit.py b/llvm/utils/lit/tests/Inputs/allowed-failures/succeeds-within-limit.py
new file mode 100644
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/allowed-failures/succeeds-within-limit.py
@@ -0,0 +1,26 @@
+# ALLOW_FAILURES: 5
+
+# This test uses a file provided externally (and stable across runs of this test)
+# to fail and only succeed the fourth time it is run.
+# RUN: "%python" "%s" "%counter"
+
+import sys
+import os
+
+counter_file = sys.argv[1]
+
+# The first time the test is run, initialize the counter to 1.
+if not os.path.exists(counter_file):
+ with open(counter_file, 'w') as counter:
+ counter.write("1")
+
+# Succeed if this is the fourth time we're being run.
+with open(counter_file, 'r') as counter:
+ num = int(counter.read())
+ if num == 4:
+ sys.exit(0)
+
+# Otherwise, increment the counter and fail
+with open(counter_file, 'w') as counter:
+ counter.write(str(num + 1))
+ sys.exit(1)
diff --git a/llvm/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt b/llvm/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
--- a/llvm/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
+++ b/llvm/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
@@ -13,6 +13,9 @@
// MY_BOOL: b)
// MY_BOOL: d
//
+// MY_INT: 4
+// MY_INT: 6
+//
// MY_BOOL_UNTERMINATED: a \
//
// END.
diff --git a/llvm/utils/lit/tests/allowed-failures.py b/llvm/utils/lit/tests/allowed-failures.py
new file mode 100644
--- /dev/null
+++ b/llvm/utils/lit/tests/allowed-failures.py
@@ -0,0 +1,17 @@
+# Check the behavior of the ALLOWED_FAILURES keyword.
+
+# RUN: rm -f %t.counter
+# RUN: %{lit} -j 1 %{inputs}/allowed-failures/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
+# CHECK-TEST1: Passes With Retry : 1
+
+# RUN: not %{lit} -j 1 %{inputs}/allowed-failures/does-not-succeed-within-limit.py | FileCheck --check-prefix=CHECK-TEST2 %s
+# CHECK-TEST2: Failing Tests (1):
+# CHECK-TEST2: allowed-failures :: does-not-succeed-within-limit.py
+
+# RUN: not %{lit} -j 1 %{inputs}/allowed-failures/more-than-one-allow-failures-lines.py | FileCheck --check-prefix=CHECK-TEST3 %s
+# CHECK-TEST3: Unresolved Tests (1):
+# CHECK-TEST3: allowed-failures :: more-than-one-allow-failures-lines.py
+
+# RUN: not %{lit} -j 1 %{inputs}/allowed-failures/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST4 %s
+# CHECK-TEST4: Unresolved Tests (1):
+# CHECK-TEST4: allowed-failures :: not-a-valid-integer.py
diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py
--- a/llvm/utils/lit/tests/unit/TestRunner.py
+++ b/llvm/utils/lit/tests/unit/TestRunner.py
@@ -57,6 +57,7 @@
IntegratedTestKeywordParser("MY_DNE_TAG.", ParserKind.TAG),
IntegratedTestKeywordParser("MY_LIST:", ParserKind.LIST),
IntegratedTestKeywordParser("MY_BOOL:", ParserKind.BOOLEAN_EXPR),
+ IntegratedTestKeywordParser("MY_INT:", ParserKind.INTEGER),
IntegratedTestKeywordParser("MY_RUN:", ParserKind.COMMAND),
IntegratedTestKeywordParser("MY_CUSTOM:", ParserKind.CUSTOM,
custom_parse),
@@ -112,6 +113,17 @@
self.assertEqual(value[0].strip(), "a && (b)")
self.assertEqual(value[1].strip(), "d")
+ def test_integer(self):
+ parsers = self.make_parsers()
+ self.parse_test(parsers)
+ int_parser = self.get_parser(parsers, 'MY_INT:')
+ value = int_parser.getValue()
+ self.assertEqual(len(value), 2) # there are only two MY_INT: lines
+ self.assertEqual(type(value[0]), int)
+ self.assertEqual(value[0], 4)
+ self.assertEqual(type(value[1]), int)
+ self.assertEqual(value[1], 6)
+
def test_boolean_unterminated(self):
parsers = self.make_parsers() + \
[IntegratedTestKeywordParser("MY_BOOL_UNTERMINATED:", ParserKind.BOOLEAN_EXPR)]