Index: utils/lit/lit/BooleanExpression.py =================================================================== --- /dev/null +++ utils/lit/lit/BooleanExpression.py @@ -0,0 +1,249 @@ +import re + +class BooleanExpression: + # A simple evaluator of boolean expressions. + # + # Grammar: + # expr :: or_expr + # or_expr :: and_expr ('||' and_expr)* + # and_expr :: not_expr ('&&' not_expr)* + # not_expr :: '!' not_expr + # '(' or_expr ')' + # identifier + # identifier :: [_a-zA-Z][-+=._a-zA-Z0-9]* + + # Evaluates `string` as a boolean expression. + # Returns True or False. Throws a ValueError on syntax error. + # + # Variables in `variables` are true. + # Substrings of `triple` are true. + # 'true' is true. + # All other identifiers are false. + @staticmethod + def evaluate(string, variables, triple=""): + try: + parser = BooleanExpression(string, set(variables), triple) + return parser.parseAll() + except ValueError as e: + raise ValueError(str(e) + ('\nin expression: %r' % string)) + + ##### + + def __init__(self, string, variables, triple=""): + self.tokens = BooleanExpression.tokenize(string) + self.variables = variables + self.variables.add('true') + self.triple = triple + self.value = None + self.token = None + + # Singleton end-of-expression marker. + END = object() + + # Tokenization pattern. + Pattern = re.compile(r'\A\s*([()]|\b[_a-zA-Z][-+=._a-zA-Z0-9]*|&&|\|\||!)\s*(.*)\Z') + + @staticmethod + def tokenize(string): + while True: + m = re.match(BooleanExpression.Pattern, string) + if m is None: + if string == "": + yield BooleanExpression.END; + return + else: + raise ValueError("couldn't parse text: %r" % string) + + token = m.group(1) + string = m.group(2) + yield token + + def quote(self, token): + if token is BooleanExpression.END: + return '' + else: + return repr(token) + + def accept(self, t): + if self.token == t: + self.token = next(self.tokens) + return True + else: + return False + + def expect(self, t): + if self.token == t: + if self.token != BooleanExpression.END: + self.token = next(self.tokens) + else: + raise ValueError("expected: %s\nhave: %s" % + (self.quote(t), self.quote(self.token))) + + def isIdentifier(self, t): + if (t is BooleanExpression.END or t == '&&' or t == '||' or + t == '!' or t == '(' or t == ')'): + return False + return True + + def parseNOT(self): + if self.accept('!'): + self.parseNOT() + self.value = not self.value + elif self.accept('('): + self.parseOR() + self.expect(')') + elif not self.isIdentifier(self.token): + raise ValueError("expected: '!' or '(' or identifier\nhave: %s" % + self.quote(self.token)) + else: + self.value = (self.token in self.variables or + self.token in self.triple) + self.token = next(self.tokens) + + def parseAND(self): + self.parseNOT() + while self.accept('&&'): + left = self.value + self.parseNOT() + right = self.value + # this is technically the wrong associativity, but it + # doesn't matter for this limited expression grammar + self.value = left and right + + def parseOR(self): + self.parseAND() + while self.accept('||'): + left = self.value + self.parseAND() + right = self.value + # this is technically the wrong associativity, but it + # doesn't matter for this limited expression grammar + self.value = left or right + + def parseAll(self): + self.token = next(self.tokens) + self.parseOR() + self.expect(BooleanExpression.END) + return self.value + + +####### +# Tests + +import unittest + +class TestBooleanExpression(unittest.TestCase): + def test_variables(self): + variables = {'its-true', 'false-lol-true', 'under_score', + 'e=quals', 'd1g1ts'} + self.assertTrue(BooleanExpression.evaluate('true', variables)) + self.assertTrue(BooleanExpression.evaluate('its-true', variables)) + self.assertTrue(BooleanExpression.evaluate('false-lol-true', variables)) + self.assertTrue(BooleanExpression.evaluate('under_score', variables)) + self.assertTrue(BooleanExpression.evaluate('e=quals', variables)) + self.assertTrue(BooleanExpression.evaluate('d1g1ts', variables)) + + self.assertFalse(BooleanExpression.evaluate('false', variables)) + self.assertFalse(BooleanExpression.evaluate('True', variables)) + self.assertFalse(BooleanExpression.evaluate('true-ish', variables)) + self.assertFalse(BooleanExpression.evaluate('not_true', variables)) + self.assertFalse(BooleanExpression.evaluate('tru', variables)) + + def test_triple(self): + triple = 'arch-vendor-os' + self.assertTrue(BooleanExpression.evaluate('arch', {}, triple)) + self.assertTrue(BooleanExpression.evaluate('ar', {}, triple)) + self.assertTrue(BooleanExpression.evaluate('ch-vend', {}, triple)) + self.assertTrue(BooleanExpression.evaluate('os', {}, triple)) + self.assertFalse(BooleanExpression.evaluate('arch-os', {}, triple)) + + def test_operators(self): + self.assertTrue(BooleanExpression.evaluate('true || true', {})) + self.assertTrue(BooleanExpression.evaluate('true || false', {})) + self.assertTrue(BooleanExpression.evaluate('false || true', {})) + self.assertFalse(BooleanExpression.evaluate('false || false', {})) + + self.assertTrue(BooleanExpression.evaluate('true && true', {})) + self.assertFalse(BooleanExpression.evaluate('true && false', {})) + self.assertFalse(BooleanExpression.evaluate('false && true', {})) + self.assertFalse(BooleanExpression.evaluate('false && false', {})) + + self.assertFalse(BooleanExpression.evaluate('!true', {})) + self.assertTrue(BooleanExpression.evaluate('!false', {})) + + self.assertTrue(BooleanExpression.evaluate(' ((!((false) )) ) ', {})) + self.assertTrue(BooleanExpression.evaluate('true && (true && (true))', {})) + self.assertTrue(BooleanExpression.evaluate('!false && !false && !! !false', {})) + self.assertTrue(BooleanExpression.evaluate('false && false || true', {})) + self.assertTrue(BooleanExpression.evaluate('(false && false) || true', {})) + self.assertFalse(BooleanExpression.evaluate('false && (false || true)', {})) + + # Evaluate boolean expression `expr`. + # Fail if it does not throw a ValueError containing the text `error`. + def checkException(self, expr, error): + try: + BooleanExpression.evaluate(expr, {}) + self.fail("expression %r didn't cause an exception" % expr) + except ValueError as e: + if -1 == str(e).find(error): + self.fail(("expression %r caused the wrong ValueError\n" + + "actual error was:\n%s\n" + + "expected error was:\n%s\n") % (expr, e, error)) + except: + self.fail("expression %r caused the wrong exception; actual exception was: \n%r" % (expr, e)) + + def test_errors(self): + self.checkException("ba#d", + "couldn't parse text: '#d'\n" + + "in expression: 'ba#d'") + + self.checkException("true and true", + "expected: \n" + + "have: 'and'\n" + + "in expression: 'true and true'") + + self.checkException("|| true", + "expected: '!' or '(' or identifier\n" + + "have: '||'\n" + + "in expression: '|| true'") + + self.checkException("true &&", + "expected: '!' or '(' or identifier\n" + + "have: \n" + + "in expression: 'true &&'") + + self.checkException("", + "expected: '!' or '(' or identifier\n" + + "have: \n" + + "in expression: ''") + + self.checkException("*", + "couldn't parse text: '*'\n" + + "in expression: '*'") + + self.checkException("no wait stop", + "expected: \n" + + "have: 'wait'\n" + + "in expression: 'no wait stop'") + + self.checkException("0-numbers-allowed", + "couldn't parse text: '0-numbers-allowed'\n" + + "in expression: '0-numbers-allowed'") + + self.checkException("(((true && true) || true)", + "expected: ')'\n" + + "have: \n" + + "in expression: '(((true && true) || true)'") + + self.checkException("true (true)", + "expected: \n" + + "have: '('\n" + + "in expression: 'true (true)'") + + self.checkException("( )", + "expected: '!' or '(' or identifier\n" + + "have: ')'\n" + + "in expression: '( )'") + +if __name__ == '__main__': + unittest.main() Index: utils/lit/lit/Test.py =================================================================== --- utils/lit/lit/Test.py +++ utils/lit/lit/Test.py @@ -2,6 +2,8 @@ from xml.sax.saxutils import escape from json import JSONEncoder +from lit.BooleanExpression import BooleanExpression + # Test result codes. class ResultCode(object): @@ -180,10 +182,24 @@ self.path_in_suite = path_in_suite self.config = config self.file_path = file_path - # A list of conditions under which this test is expected to fail. These - # can optionally be provided by test format handlers, and will be - # honored when the test result is supplied. + + # A list of conditions under which this test is expected to fail. + # Each condition is a boolean expression of features and target + # triple parts. These can optionally be provided by test format + # handlers, and will be honored when the test result is supplied. self.xfails = [] + + # A list of conditions that must be satisfied before running the test. + # Each condition is a boolean expression of features. All of them + # must be True for the test to run. + # FIXME should target triple parts count here too? + self.requires = [] + + # A list of conditions that prevent execution of the test. + # Each condition is a boolean expression of features and target + # triple parts. All of them must be False for the test to run. + self.unsupported = [] + # The test result, once complete. self.result = None @@ -196,11 +212,16 @@ self.result = result # Apply the XFAIL handling to resolve the result exit code. - if self.isExpectedToFail(): - if self.result.code == PASS: - self.result.code = XPASS - elif self.result.code == FAIL: - self.result.code = XFAIL + try: + if self.isExpectedToFail(): + if self.result.code == PASS: + self.result.code = XPASS + elif self.result.code == FAIL: + self.result.code = XFAIL + except ValueError as e: + # Syntax error in an XFAIL line. + self.result.code = UNRESOLVED + self.result.output = str(e) def getFullName(self): return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) @@ -224,24 +245,91 @@ configuration. This check relies on the test xfails property which by some test formats may not be computed until the test has first been executed. + Throws ValueError if an XFAIL line has a syntax error. """ + features = self.config.available_features + triple = getattr(self.suite.config, 'target_triple', "") + # Check if any of the xfails match an available feature or the target. for item in self.xfails: # If this is the wildcard, it always fails. if item == '*': return True - # If this is an exact match for one of the features, it fails. - if item in self.config.available_features: - return True - - # If this is a part of the target triple, it fails. - if item and item in self.suite.config.target_triple: - return True + # If this is a True expression of features and target triple parts, + # it fails. + try: + if BooleanExpression.evaluate(item, features, triple): + return True + except ValueError as e: + raise ValueError('Error in XFAIL list:\n%s' % str(e)) return False + def isWithinFeatureLimits(self): + """ + isWithinFeatureLimits() -> bool + + A test is within the feature limits set by run_only_tests if + 1. the test's requirements ARE satisfied by the available features + 2. the test's requirements ARE NOT satisfied after the limiting + features are removed from the available features + + Throws ValueError if a REQUIRES line has a syntax error. + """ + + if not self.config.limit_to_features: + return True # No limits. Run it. + + # Check the requirements as-is (#1) + if self.getMissingRequiredFeatures(): + return False + + # Check the requirements after removing the limiting features (#2) + featuresMinusLimits = [f for f in self.config.available_features + if not f in self.config.limit_to_features] + if not self.getMissingRequiredFeaturesFromList(featuresMinusLimits): + return False + + return True + + def getMissingRequiredFeaturesFromList(self, features): + try: + return [item for item in self.requires + if not BooleanExpression.evaluate(item, features)] + except ValueError as e: + raise ValueError('Error in REQUIRES list:\n%s' % str(e)) + + def getMissingRequiredFeatures(self): + """ + getMissingRequiredFeatures() -> list of strings + + Returns a list of features from REQUIRES that are not satisfied." + Throws ValueError if a REQUIRES line has a syntax error. + """ + + features = self.config.available_features + return self.getMissingRequiredFeaturesFromList(features) + + def getUnsupportedFeatures(self): + """ + getUnsupportedFeatures() -> list of strings + + Returns a list of features from UNSUPPORTED that are present + in the test configuration's features or target triple. + Throws ValueError if an UNSUPPORTED line has a syntax error. + """ + + features = self.config.available_features + triple = getattr(self.suite.config, 'target_triple', "") + + try: + return [item for item in self.unsupported + if BooleanExpression.evaluate(item, features, triple)] + except ValueError as e: + raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e)) + def isEarlyTest(self): """ isEarlyTest() -> bool Index: utils/lit/lit/TestRunner.py =================================================================== --- utils/lit/lit/TestRunner.py +++ utils/lit/lit/TestRunner.py @@ -682,9 +682,6 @@ # Collect the test lines from the script. sourcepath = test.getSourcePath() script = [] - requires = [] - requires_any = [] - unsupported = [] keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'REQUIRES-ANY:', 'UNSUPPORTED:', 'END.'] for line_number, command_type, ln in \ @@ -710,11 +707,9 @@ elif command_type == 'XFAIL': test.xfails.extend([s.strip() for s in ln.split(',')]) elif command_type == 'REQUIRES': - requires.extend([s.strip() for s in ln.split(',')]) - elif command_type == 'REQUIRES-ANY': - requires_any.extend([s.strip() for s in ln.split(',')]) + test.requires.extend([s.strip() for s in ln.split(',')]) elif command_type == 'UNSUPPORTED': - unsupported.extend([s.strip() for s in ln.split(',')]) + test.unsupported.extend([s.strip() for s in ln.split(',')]) elif command_type == 'END': # END commands are only honored if the rest of the line is empty. if not ln.strip(): @@ -732,41 +727,25 @@ return lit.Test.Result(Test.UNRESOLVED, "Test has unterminated run lines (with '\\')") - # Check that we have the required features: - missing_required_features = [f for f in requires - if f not in test.config.available_features] + # Enforce REQUIRES + missing_required_features = test.getMissingRequiredFeatures() if missing_required_features: msg = ', '.join(missing_required_features) return lit.Test.Result(Test.UNSUPPORTED, - "Test requires the following features: %s" % msg) - requires_any_features = [f for f in requires_any - if f in test.config.available_features] - if requires_any and not requires_any_features: - msg = ' ,'.join(requires_any) - return lit.Test.Result(Test.UNSUPPORTED, - "Test requires any of the following features: %s" % msg) - unsupported_features = [f for f in unsupported - if f in test.config.available_features] + "Test requires the following unavailable features: %s" % msg) + + # Enforce UNSUPPORTED + unsupported_features = test.getUnsupportedFeatures() if unsupported_features: msg = ', '.join(unsupported_features) return lit.Test.Result(Test.UNSUPPORTED, - "Test is unsupported with the following features: %s" % msg) + "Test does not support the following features and/or targets: %s" % msg) - unsupported_targets = [f for f in unsupported - if f in test.suite.config.target_triple] - if unsupported_targets: - return lit.Test.Result(Test.UNSUPPORTED, - "Test is unsupported with the following triple: %s" % ( - test.suite.config.target_triple,)) - - if test.config.limit_to_features: - # Check that we have one of the limit_to_features features in requires. - limit_to_features_tests = [f for f in test.config.limit_to_features - if f in requires] - if not limit_to_features_tests: - msg = ', '.join(test.config.limit_to_features) - return lit.Test.Result(Test.UNSUPPORTED, - "Test requires one of the limit_to_features features %s" % msg) + # Enforce limit_to_features + if not test.isWithinFeatureLimits(): + msg = ', '.join(test.config.limit_to_features) + return lit.Test.Result(Test.UNSUPPORTED, + "Test does not require any of the features specified in limit_to_features: %s" % msg) return script Index: utils/lit/tests/Inputs/shtest-format/requires-any-missing.txt =================================================================== --- utils/lit/tests/Inputs/shtest-format/requires-any-missing.txt +++ /dev/null @@ -1,2 +0,0 @@ -RUN: true -REQUIRES-ANY: a-missing-feature, a-missing-feature-2 Index: utils/lit/tests/Inputs/shtest-format/requires-any-present.txt =================================================================== --- utils/lit/tests/Inputs/shtest-format/requires-any-present.txt +++ /dev/null @@ -1,2 +0,0 @@ -RUN: true -REQUIRES-ANY: a-missing-feature, a-present-feature Index: utils/lit/tests/Inputs/shtest-format/requires-missing.txt =================================================================== --- utils/lit/tests/Inputs/shtest-format/requires-missing.txt +++ utils/lit/tests/Inputs/shtest-format/requires-missing.txt @@ -1,2 +1,5 @@ -RUN: true -REQUIRES: a-missing-feature +# REQUIRES with a false clause. Test should not run. +REQUIRES: true +REQUIRES: a-missing-feature, true +REQUIRES: true +RUN: false Index: utils/lit/tests/Inputs/shtest-format/requires-present.txt =================================================================== --- utils/lit/tests/Inputs/shtest-format/requires-present.txt +++ utils/lit/tests/Inputs/shtest-format/requires-present.txt @@ -1,2 +1,4 @@ +# REQUIRES with only true clauses. Test should run. +REQUIRES: a-present-feature, true, !not-true +REQUIRES: true RUN: true -REQUIRES: a-present-feature Index: utils/lit/tests/Inputs/shtest-format/requires-star.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/requires-star.txt @@ -0,0 +1,3 @@ +# '*' only works in XFAIL +REQUIRES: * +RUN: false Index: utils/lit/tests/Inputs/shtest-format/requires-triple.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/requires-triple.txt @@ -0,0 +1,3 @@ +# REQUIRES line that uses target triple, which doesn't work. Test should not run +REQUIRES: x86_64 +RUN: false Index: utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/unsupported-expr-false.txt @@ -0,0 +1,9 @@ +# UNSUPPORTED with only false clauses. Test should run. +UNSUPPORTED: false +UNSUPPORTED: false, not-true +UNSUPPORTED: false +UNSUPPORTED: still-not-true +UNSUPPORTED: false +UNSUPPORTED: false +UNSUPPORTED: false +RUN: true Index: utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/unsupported-expr-true.txt @@ -0,0 +1,4 @@ +# UNSUPPORTED with a true clause. Test should not run. +UNSUPPORTED: false +UNSUPPORTED: false, false, false, _64-unk && a-present-feature, false +RUN: false Index: utils/lit/tests/Inputs/shtest-format/unsupported-star.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/unsupported-star.txt @@ -0,0 +1,3 @@ +# '*' only works in XFAIL +UNSUPPORTED: * +RUN: false Index: utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/xfail-expr-false.txt @@ -0,0 +1,3 @@ +# XFAIL with only false clauses. Test should run. +XFAIL: false, a-missing-feature || ! a-present-feature || ! x86_64, false +RUN: true Index: utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt =================================================================== --- /dev/null +++ utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt @@ -0,0 +1,4 @@ +# XFAIL with a true clause. Test should not run. +XFAIL: false +XFAIL: false, a-present-feature && ! a-missing-feature && x86_64 +RUN: false Index: utils/lit/tests/boolean-parsing.py =================================================================== --- /dev/null +++ utils/lit/tests/boolean-parsing.py @@ -0,0 +1,4 @@ +# Test the boolean expression parser +# used for REQUIRES and UNSUPPORTED and XFAIL + +# RUN: %{python} -m lit.BooleanExpression Index: utils/lit/tests/shtest-format.py =================================================================== --- utils/lit/tests/shtest-format.py +++ utils/lit/tests/shtest-format.py @@ -46,11 +46,16 @@ # CHECK: UNRESOLVED: shtest-format :: no-test-line.txt # CHECK: PASS: shtest-format :: pass.txt -# CHECK: UNSUPPORTED: shtest-format :: requires-any-missing.txt -# CHECK: PASS: shtest-format :: requires-any-present.txt # CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt # CHECK: PASS: shtest-format :: requires-present.txt +# CHECK: UNRESOLVED: shtest-format :: requires-star.txt +# CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt +# CHECK: PASS: shtest-format :: unsupported-expr-false.txt +# CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt +# CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt # CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt +# CHECK: PASS: shtest-format :: xfail-expr-false.txt +# CHECK: XFAIL: shtest-format :: xfail-expr-true.txt # CHECK: XFAIL: shtest-format :: xfail-feature.txt # CHECK: XFAIL: shtest-format :: xfail-target.txt # CHECK: XFAIL: shtest-format :: xfail.txt @@ -70,9 +75,9 @@ # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt # CHECK: shtest-format :: fail.txt -# CHECK: Expected Passes : 5 -# CHECK: Expected Failures : 3 -# CHECK: Unsupported Tests : 3 -# CHECK: Unresolved Tests : 1 +# CHECK: Expected Passes : 6 +# CHECK: Expected Failures : 4 +# CHECK: Unsupported Tests : 4 +# CHECK: Unresolved Tests : 3 # CHECK: Unexpected Passes : 1 # CHECK: Unexpected Failures: 3