Index: utils/lit/lit/BooleanExpression.py
===================================================================
--- utils/lit/lit/BooleanExpression.py
+++ utils/lit/lit/BooleanExpression.py
@@ -0,0 +1,126 @@
+import re
+
+class BooleanExpression:
+ # A simple evaluator of boolean expressions.
+ #
+ # Grammar:
+ # expr :: or_expr
+ # or_expr :: and_expr ('or' and_expr)*
+ # and_expr :: not_expr ('and' not_expr)*
+ # not_expr :: 'not' not_expr
+ # '(' or_expr ')'
+ # identifier
+ # identifier :: [a-zA-Z_][-=:/_a-zA-Z0-9]*
+
+ # Evaluates `string` as a boolean expression.
+ # Returns True or False. Throws a ValueError on syntax error.
+ #
+ # Variables in `variables` are True.
+ # Substrings of `triple` are True.
+ # 'True' is True.
+ # All other identifiers are False.
+ @staticmethod
+ def evaluate(string, variables, triple=""):
+ try:
+ parser = BooleanExpression(string, set(variables), triple)
+ return parser.parseAll()
+ except ValueError as e:
+ raise ValueError(str(e) + ('\nin expression: %r' % string))
+
+ #####
+
+ def __init__(self, string, variables, triple=""):
+ self.tokens = BooleanExpression.tokenize(string)
+ self.variables = variables
+ self.variables.add('True')
+ self.triple = triple
+ self.value = None
+ self.token = None
+
+ # Singleton end-of-expression marker.
+ END = object()
+
+ @staticmethod
+ def tokenize(string):
+ pattern = re.compile(r'\A\s*([()]|\b[_a-zA-Z][-=:/_a-zA-Z0-9]*)\s*(.*)\Z')
+ while True:
+ m = re.match(pattern, string)
+ if m is None:
+ if string == "":
+ yield BooleanExpression.END;
+ return
+ else:
+ raise ValueError("couldn't parse text: %r" % string)
+
+ token = m.group(1)
+ string = m.group(2)
+ yield token
+
+ def quote(self, token):
+ if token is BooleanExpression.END:
+ return '<end of expression>'
+ else:
+ return repr(token)
+
+ def accept(self, t):
+ if self.token == t:
+ self.token = next(self.tokens)
+ return True
+ else:
+ return False
+
+ def expect(self, t):
+ if self.token == t:
+ if self.token != BooleanExpression.END:
+ self.token = next(self.tokens)
+ else:
+ raise ValueError("expected: %s\nhave: %s" %
+ (self.quote(t), self.quote(self.token)))
+
+ def isIdentifier(self, t):
+ if (t is BooleanExpression.END or t == 'and' or t == 'or' or
+ t == 'not' or t == '(' or t == ')'):
+ return False
+ return True
+
+ def parseNOT(self):
+ if self.accept('not'):
+ self.parseNOT()
+ self.value = not self.value
+ elif self.accept('('):
+ self.parseOR()
+ self.expect(')')
+ elif not self.isIdentifier(self.token):
+ raise ValueError("expected: '!' or '(' or identifier\nhave: %s" %
+ self.quote(self.token))
+ else:
+ self.value = (self.token in self.variables or
+ self.token in self.triple)
+ self.token = next(self.tokens)
+
+ def parseAND(self):
+ self.parseNOT()
+ while self.accept('and'):
+ left = self.value
+ self.parseNOT()
+ right = self.value
+ # this is technically the wrong associativity, but it
+ # doesn't matter for this limited expression grammar
+ self.value = left and right
+
+ def parseOR(self):
+ self.parseAND()
+ while self.accept('or'):
+ left = self.value
+ self.parseAND()
+ right = self.value
+ # this is technically the wrong associativity, but it
+ # doesn't matter for this limited expression grammar
+ self.value = left or right
+
+ def parseAll(self):
+ self.token = next(self.tokens)
+ self.parseOR()
+ self.expect(BooleanExpression.END)
+ return self.value
+
Index: utils/lit/lit/Test.py
===================================================================
--- utils/lit/lit/Test.py
+++ utils/lit/lit/Test.py
@@ -2,6 +2,8 @@
from xml.sax.saxutils import escape
from json import JSONEncoder
+from BooleanExpression import BooleanExpression
+
# Test result codes.
class ResultCode(object):
@@ -173,10 +175,24 @@
self.path_in_suite = path_in_suite
self.config = config
self.file_path = file_path
- # A list of conditions under which this test is expected to fail. These
- # can optionally be provided by test format handlers, and will be
- # honored when the test result is supplied.
+
+ # A list of conditions under which this test is expected to fail.
+ # Each condition is a boolean expression of features and target
+ # triple parts. These can optionally be provided by test format
+ # handlers, and will be honored when the test result is supplied.
self.xfails = []
+
+ # A list of conditions that must be satisfied before running the test.
+ # Each condition is a boolean expression of features. All of them
+ # must be True for the test to run.
+ # FIXME should target triple parts count here too?
+ self.requires = []
+
+ # A list of conditions that prevent execution of the test.
+ # Each condition is a boolean expression of features and target
+ # triple parts. All of them must be False for the test to run.
+ self.unsupported = []
+
# The test result, once complete.
self.result = None
@@ -189,11 +205,16 @@
self.result = result
# Apply the XFAIL handling to resolve the result exit code.
- if self.isExpectedToFail():
- if self.result.code == PASS:
- self.result.code = XPASS
- elif self.result.code == FAIL:
- self.result.code = XFAIL
+ try:
+ if self.isExpectedToFail():
+ if self.result.code == PASS:
+ self.result.code = XPASS
+ elif self.result.code == FAIL:
+ self.result.code = XFAIL
+ except ValueError as e:
+ # Syntax error in an XFAIL line.
+ self.result.code = UNRESOLVED
+ self.result.output = str(e)
def getFullName(self):
return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
@@ -209,6 +230,16 @@
def getExecPath(self):
return self.suite.getExecPath(self.path_in_suite)
+ def isEarlyTest(self):
+ """
+ isEarlyTest() -> bool
+
+ Check whether this test should be executed early in a particular run.
+ This can be used for test suites with long running tests to maximize
+ parallelism or where it is desirable to surface their failures early.
+ """
+ return self.suite.config.is_early
+
def isExpectedToFail(self):
"""
isExpectedToFail() -> bool
@@ -217,24 +248,91 @@
configuration. This check relies on the test xfails property which by
some test formats may not be computed until the test has first been
executed.
+ Throws ValueError if an XFAIL line has a syntax error.
"""
+ features = self.config.available_features
+ triple = getattr(self.suite.config, 'target_triple', "")
+
# Check if any of the xfails match an available feature or the target.
for item in self.xfails:
# If this is the wildcard, it always fails.
if item == '*':
return True
- # If this is an exact match for one of the features, it fails.
- if item in self.config.available_features:
- return True
-
- # If this is a part of the target triple, it fails.
- if item and item in self.suite.config.target_triple:
- return True
+ # If this is a True expression of features and target triple parts,
+ # it fails.
+ try:
+ if BooleanExpression.evaluate(item, features, triple):
+ return True
+ except ValueError as e:
+ raise ValueError('Error in XFAIL list:\n%s' % str(e))
return False
+ def isWithinFeatureLimits(self):
+ """
+ isWithinFeatureLimits() -> bool
+
+ A test is within the feature limits set by run_only_tests if
+ 1. the test's requirements ARE satisfied by the available features
+ 2. the test's requirements ARE NOT satisfied after the limiting
+ features are removed from the available features
+
+ Throws ValueError if a REQUIRES line has a syntax error.
+ """
+
+ if not self.config.limit_to_features:
+ return True # No limits. Run it.
+
+ # Check the requirements as-is (#1)
+ if self.getMissingRequiredFeatures():
+ return False
+
+ # Check the requirements after removing the limiting features (#2)
+ featuresMinusLimits = [f for f in self.config.available_features
+ if not f in self.config.limit_to_features]
+ if not self.getMissingRequiredFeaturesFromList(featuresMinusLimits):
+ return False
+
+ return True
+
+ def getMissingRequiredFeaturesFromList(self, features):
+ try:
+ return [item for item in self.requires
+ if not BooleanExpression.evaluate(item, features)]
+ except ValueError as e:
+ raise ValueError('Error in REQUIRES list:\n%s' % str(e))
+
+ def getMissingRequiredFeatures(self):
+ """
+ getMissingRequiredFeatures() -> list of strings
+
+ Returns a list of features from REQUIRES that are not satisfied."
+ Throws ValueError if a REQUIRES line has a syntax error.
+ """
+
+ features = self.config.available_features
+ return self.getMissingRequiredFeaturesFromList(features)
+
+ def getUnsupportedFeatures(self):
+ """
+ getUnsupportedFeatures() -> list of strings
+
+ Returns a list of features from UNSUPPORTED that are present
+ in the test configuration's features or target triple.
+ Throws ValueError if an UNSUPPORTED line has a syntax error.
+ """
+
+ features = self.config.available_features
+ triple = getattr(self.suite.config, 'target_triple', "")
+
+ try:
+ return [item for item in self.unsupported
+ if BooleanExpression.evaluate(item, features, triple)]
+ except ValueError as e:
+ raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e))
+
def isEarlyTest(self):
"""
isEarlyTest() -> bool
Index: utils/lit/lit/TestRunner.py
===================================================================
--- utils/lit/lit/TestRunner.py
+++ utils/lit/lit/TestRunner.py
@@ -597,8 +597,6 @@
# Collect the test lines from the script.
sourcepath = test.getSourcePath()
script = []
- requires = []
- unsupported = []
keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.']
for line_number, command_type, ln in \
parseIntegratedTestScriptCommands(sourcepath, keywords):
@@ -623,9 +621,9 @@
elif command_type == 'XFAIL':
test.xfails.extend([s.strip() for s in ln.split(',')])
elif command_type == 'REQUIRES':
- requires.extend([s.strip() for s in ln.split(',')])
+ test.requires.extend([s.strip() for s in ln.split(',')])
elif command_type == 'UNSUPPORTED':
- unsupported.extend([s.strip() for s in ln.split(',')])
+ test.unsupported.extend([s.strip() for s in ln.split(',')])
elif command_type == 'END':
# END commands are only honored if the rest of the line is empty.
if not ln.strip():
@@ -643,35 +641,25 @@
return lit.Test.Result(Test.UNRESOLVED,
"Test has unterminated run lines (with '\\')")
- # Check that we have the required features:
- missing_required_features = [f for f in requires
- if f not in test.config.available_features]
+ # Enforce REQUIRES
+ missing_required_features = test.getMissingRequiredFeatures()
if missing_required_features:
msg = ', '.join(missing_required_features)
return lit.Test.Result(Test.UNSUPPORTED,
- "Test requires the following features: %s" % msg)
- unsupported_features = [f for f in unsupported
- if f in test.config.available_features]
+ "Test requires the following unavailable features: %s" % msg)
+
+ # Enforce UNSUPPORTED
+ unsupported_features = test.getUnsupportedFeatures()
if unsupported_features:
msg = ', '.join(unsupported_features)
return lit.Test.Result(Test.UNSUPPORTED,
- "Test is unsupported with the following features: %s" % msg)
+ "Test does not support the following features and/or targets: %s" % msg)
- unsupported_targets = [f for f in unsupported
- if f in test.suite.config.target_triple]
- if unsupported_targets:
- return lit.Test.Result(Test.UNSUPPORTED,
- "Test is unsupported with the following triple: %s" % (
- test.suite.config.target_triple,))
-
- if test.config.limit_to_features:
- # Check that we have one of the limit_to_features features in requires.
- limit_to_features_tests = [f for f in test.config.limit_to_features
- if f in requires]
- if not limit_to_features_tests:
- msg = ', '.join(test.config.limit_to_features)
- return lit.Test.Result(Test.UNSUPPORTED,
- "Test requires one of the limit_to_features features %s" % msg)
+ # Enforce limit_to_features
+ if not test.isWithinFeatureLimits():
+ msg = ', '.join(test.config.limit_to_features)
+ return lit.Test.Result(Test.UNSUPPORTED,
+ "Test does not require any of the features specified in limit_to_features: %s" % msg)
return script
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-01.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-01.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-01.txt
@@ -0,0 +1,2 @@
+UNSUPPORTED: True and
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-02.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-02.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-02.txt
@@ -0,0 +1,2 @@
+REQUIRES: True,
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-03.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-03.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-03.txt
@@ -0,0 +1,2 @@
+XFAIL: ba!d
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-04.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-04.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-04.txt
@@ -0,0 +1,2 @@
+REQUIRES: True && True
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-05.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-05.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-05.txt
@@ -0,0 +1,2 @@
+XFAIL: * or True
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-06.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-06.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-06.txt
@@ -0,0 +1,2 @@
+REQUIRES: or True
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-07.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-07.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-07.txt
@@ -0,0 +1,2 @@
+XFAIL:
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-08.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-08.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-08.txt
@@ -0,0 +1,2 @@
+REQUIRES: *
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-09.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-09.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-09.txt
@@ -0,0 +1,2 @@
+UNSUPPORTED: *
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-10.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-10.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-10.txt
@@ -0,0 +1,2 @@
+REQUIRES: no wait stop
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-11.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-11.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-11.txt
@@ -0,0 +1,2 @@
+REQUIRES: 0-numbers-allowed
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-12.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-12.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-12.txt
@@ -0,0 +1,2 @@
+REQUIRES: (((True and True) or True)
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-13.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-13.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-13.txt
@@ -0,0 +1,2 @@
+REQUIRES: True (True)
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/expr-bad-14.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/expr-bad-14.txt
+++ utils/lit/tests/Inputs/shtest-format/expr-bad-14.txt
@@ -0,0 +1,3 @@
+REQUIRES: ( )
+RUN: false
+
Index: utils/lit/tests/Inputs/shtest-format/lit.cfg
===================================================================
--- utils/lit/tests/Inputs/shtest-format/lit.cfg
+++ utils/lit/tests/Inputs/shtest-format/lit.cfg
@@ -6,3 +6,4 @@
config.test_exec_root = None
config.target_triple = 'x86_64-unknown-unknown'
config.available_features.add('a-present-feature')
+config.available_features.add('another-present-feature')
Index: utils/lit/tests/Inputs/shtest-format/requires-missing.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/requires-missing.txt
+++ utils/lit/tests/Inputs/shtest-format/requires-missing.txt
@@ -1,2 +1,5 @@
-RUN: true
-REQUIRES: a-missing-feature
+# REQUIRES with a False clause. Test should not run.
+REQUIRES: True
+REQUIRES: a-missing-feature, True
+REQUIRES: True
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/requires-present.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/requires-present.txt
+++ utils/lit/tests/Inputs/shtest-format/requires-present.txt
@@ -1,2 +1,7 @@
-RUN: true
+# REQUIRES with only True clauses. Test should run.
REQUIRES: a-present-feature
+REQUIRES: a-present-feature and True and not False and another-present-feature
+REQUIRES:True
+REQUIRES:True,(True),(((((True))))),True and True and True and True and True
+REQUIRES: (False or ( not True) ) or True
+RUN: true
Index: utils/lit/tests/Inputs/shtest-format/requires-triple.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/requires-triple.txt
+++ utils/lit/tests/Inputs/shtest-format/requires-triple.txt
@@ -0,0 +1,3 @@
+# REQUIRES line that uses target triple, which doesn't work. Test should not run
+REQUIRES: x86_64
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
+++ utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt
@@ -0,0 +1,9 @@
+# UNSUPPORTED with only False clauses. Test should run.
+UNSUPPORTED: False
+UNSUPPORTED: False, True-ish, not True, x86_65, False and False, False or False
+UNSUPPORTED: False
+UNSUPPORTED: False
+UNSUPPORTED: False
+UNSUPPORTED: False
+UNSUPPORTED: False
+RUN: true
Index: utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
+++ utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt
@@ -0,0 +1,4 @@
+# UNSUPPORTED with a True clause. Test should not run.
+UNSUPPORTED: False
+UNSUPPORTED: False, False, False, _64-unk and a-present-feature, False
+RUN: false
Index: utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
+++ utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt
@@ -0,0 +1,3 @@
+# XFAIL with only False clauses. Test should run.
+XFAIL: False, a-missing-feature or not a-present-feature or not x86_64, False
+RUN: true
Index: utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
===================================================================
--- utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
+++ utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
@@ -0,0 +1,4 @@
+# XFAIL with a True clause. Test should not run.
+XFAIL: False
+XFAIL: False, a-present-feature and not a-missing-feature and x86_64
+RUN: false
Index: utils/lit/tests/shtest-format.py
===================================================================
--- utils/lit/tests/shtest-format.py
+++ utils/lit/tests/shtest-format.py
@@ -8,6 +8,75 @@
# CHECK: -- Testing:
# CHECK: PASS: shtest-format :: argv0.txt
+
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-01.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-01.txt' FAILED ***
+# CHECK: Error in UNSUPPORTED list
+# CHECK: expected: '!' or '(' or identifier
+# CHECK: have: <end of expression>
+# CHECK: in expression: 'True and'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-02.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-02.txt' FAILED ***
+# CHECK: Error in REQUIRES list
+# CHECK: expected: '!' or '(' or identifier
+# CHECK: have: <end of expression>
+# CHECK: in expression: ''
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-03.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-03.txt' FAILED ***
+# CHECK: Error in XFAIL list
+# CHECK: couldn't parse text: '!d'
+# CHECK: in expression: 'ba!d'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-04.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-04.txt' FAILED ***
+# CHECK: couldn't parse text: '&& True'
+# CHECK: in expression: 'True && True'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-05.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-05.txt' FAILED ***
+# CHECK: couldn't parse text: '* or True'
+# CHECK: in expression: '* or True'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-06.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-06.txt' FAILED ***
+# CHECK: expected: '!' or '(' or identifier
+# CHECK: have: 'or'
+# CHECK: in expression: 'or True'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-07.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-07.txt' FAILED ***
+# CHECK: expected: '!' or '(' or identifier
+# CHECK: have: <end of expression>
+# CHECK: in expression: ''
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-08.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-08.txt' FAILED ***
+# CHECK: couldn't parse text: '*'
+# CHECK: in expression: '*'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-09.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-09.txt' FAILED ***
+# CHECK: couldn't parse text: '*'
+# CHECK: in expression: '*'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-10.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-10.txt' FAILED ***
+# CHECK: expected: <end of expression>
+# CHECK: have: 'wait'
+# CHECK: in expression: 'no wait stop'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-11.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-11.txt' FAILED ***
+# CHECK: couldn't parse text: '0-numbers-allowed'
+# CHECK: in expression: '0-numbers-allowed'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-12.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-12.txt' FAILED ***
+# CHECK: expected: ')'
+# CHECK: have: <end of expression>
+# CHECK: in expression: '(((True and True) or True)'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-13.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-13.txt' FAILED ***
+# CHECK: expected: <end of expression>
+# CHECK: have: '('
+# CHECK: in expression: 'True (True)'
+# CHECK: UNRESOLVED: shtest-format :: expr-bad-14.txt
+# CHECK-NEXT: *** TEST 'shtest-format :: expr-bad-14.txt' FAILED ***
+# CHECK: expected: '!' or '(' or identifier
+# CHECK: have: ')'
+# CHECK: in expression: '( )'
+
# CHECK: FAIL: shtest-format :: external_shell/fail.txt
# CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail.txt' FAILED ***
# CHECK: Command Output (stdout):
@@ -49,7 +118,12 @@
# CHECK: PASS: shtest-format :: pass.txt
# CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
# CHECK: PASS: shtest-format :: requires-present.txt
+# CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
+# CHECK: PASS: shtest-format :: unsupported-expr-false.txt
+# CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
# CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
+# CHECK: PASS: shtest-format :: xfail-expr-false.txt
+# CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
# CHECK: XFAIL: shtest-format :: xfail-feature.txt
# CHECK: XFAIL: shtest-format :: xfail-target.txt
# CHECK: XFAIL: shtest-format :: xfail.txt
@@ -69,9 +143,9 @@
# CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
# CHECK: shtest-format :: fail.txt
-# CHECK: Expected Passes : 4
-# CHECK: Expected Failures : 3
-# CHECK: Unsupported Tests : 2
-# CHECK: Unresolved Tests : 1
+# CHECK: Expected Passes : 6
+# CHECK: Expected Failures : 4
+# CHECK: Unsupported Tests : 4
+# CHECK: Unresolved Tests : 15
# CHECK: Unexpected Passes : 1
# CHECK: Unexpected Failures: 3