[lit] Add --xfail-not/LIT_XFAIL_NOT

[jdenny]

Summary

For example, I need this lately in my CI config:

LIT_XFAIL_NOT='libomptarget :: nvptx64-nvidia-cuda :: unified_shared_memory/api.c'

That test specifies an XFAIL directive, but I get an XPASS result.

[jdenny]

llvm/docs/CommandGuide/lit.rst

272

Sorry. Will fix later.

[jhenderson]

llvm/docs/CommandGuide/lit.rst

270

References to options should be preceded with :option:, so that they're auto-linked to them. Take a look at e.g. some of the LLVM binutils for examples of its usage.

272

I don't think you need the "However" bit of the last sentence (the rest of it is fine).

llvm/utils/lit/tests/xfail-cl.py

13

?

[jdenny]

Apply suggestions from review.

@jhenderson: Thanks!

[jhenderson]

LGTM.

llvm/docs/CommandGuide/lit.rst

.. option:: --xfail=LIST
   reported in LIT output.  For example, we can adjust the previous
   example not to treat the ``nvptx64-nvidia-cuda`` version of
   ``offloading/memory_manager.cpp`` as XFAIL:
 
   .. code-block:: none
 
     LIT_XFAIL="affinity/kmp-hw-subset.c;libomptarget :: x86_64-pc-linux-gnu :: offloading/memory_manager.cpp"
 
+.. option:: --xfail-not=LIST
+
+  Do not treat the specified tests as ``XFAIL``.  The environment variable
+  ``LIT_XFAIL_NOT`` can also be used in place of this option.  The syntax is the
+  same as for ``--xfail`` and ``LIT_XFAIL``.  ``--xfail-not`` and
+  ``LIT_XFAIL_NOT`` always override all other ``XFAIL`` specifications,
+  including an ``--xfail-not`` appearing later on the command line.  However,
+  the primary purpose is to suppress an ``XPASS`` result without modifying a
+  test case that uses the ``XFAIL`` directive.
 
 ADDITIONAL OPTIONS
 ------------------
 
 .. option:: --debug
 
  Run :program:`lit` in debug mode, for debugging configuration issues and
  :program:`lit` itself.

llvm/utils/lit/lit/Test.py

def __init__(self, suite, path_in_suite, config, file_path = None):
         self.file_path = file_path
 
         # A list of conditions under which this test is expected to fail.
         # Each condition is a boolean expression of features and target
         # triple parts. These can optionally be provided by test format
         # handlers, and will be honored when the test result is supplied.
         self.xfails = []
 
+        # If true, ignore all items in self.xfails.
+        self.xfail_not = False
+
         # A list of conditions that must be satisfied before running the test.
         # Each condition is a boolean expression of features. All of them
         # must be True for the test to run.
         # FIXME should target triple parts count here too?
         self.requires = []
 
         # A list of conditions that prevent execution of the test.
         # Each condition is a boolean expression of features and target
def isExpectedToFail(self):
 
         Check whether this test is expected to fail in the current
         configuration. This check relies on the test xfails property which by
         some test formats may not be computed until the test has first been
         executed.
         Throws ValueError if an XFAIL line has a syntax error.
         """
 
+        if self.xfail_not:
+          return False
+
         features = self.config.available_features
         triple = getattr(self.suite.config, 'target_triple', "")
 
         # Check if any of the xfails match an available feature or the target.
         for item in self.xfails:
             # If this is the wildcard, it always fails.
             if item == '*':
                 return True

llvm/utils/lit/lit/cl_arguments.py

selection_group.add_argument("--filter-out",
             type=_case_insensitive_regex,
             help="Filter out tests with paths matching the given regular expression",
             default=os.environ.get("LIT_FILTER_OUT", "^$"))
     selection_group.add_argument("--xfail",
             metavar="LIST",
             type=_semicolon_list,
             help="XFAIL tests with paths in the semicolon separated list",
             default=os.environ.get("LIT_XFAIL", ""))
+    selection_group.add_argument("--xfail-not",
+            metavar="LIST",
+            type=_semicolon_list,
+            help="do not XFAIL tests with paths in the semicolon separated list",
+            default=os.environ.get("LIT_XFAIL_NOT", ""))
     selection_group.add_argument("--num-shards",
             dest="numShards",
             metavar="M",
             help="Split testsuite into M pieces and only run one",
             type=_positive_int,
             default=os.environ.get("LIT_NUM_SHARDS"))
     selection_group.add_argument("--run-shard",
             dest="runShard",

llvm/utils/lit/lit/main.py

 
 
 def mark_xfail(selected_tests, opts):
     for t in selected_tests:
         test_file = os.sep.join(t.path_in_suite)
         test_full_name = t.getFullName()
         if test_file in opts.xfail or test_full_name in opts.xfail:
             t.xfails += '*'
+        if test_file in opts.xfail_not or test_full_name in opts.xfail_not:
+            t.xfail_not = True
 
 def mark_excluded(discovered_tests, selected_tests):
     excluded_tests = set(discovered_tests) - set(selected_tests)
     result = lit.Test.Result(lit.Test.EXCLUDED)
     for t in excluded_tests:
         t.setResult(result)
 
 

llvm/utils/lit/tests/Inputs/xfail-cl/a/test-xfail.txt

+# XFAIL: *
+# RUN: true

llvm/utils/lit/tests/Inputs/xfail-cl/b/test-xfail.txt

+# XFAIL: *
+# RUN: false

llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail.txt

+# XFAIL: *
+# RUN: true

llvm/utils/lit/tests/xfail-cl.py

 # Check that XFAILing works via command line or env var.
 
 # RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
-# RUN:        %{inputs}/xfail-cl \
+# RUN:   --xfail-not 'true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
+# RUN:   %{inputs}/xfail-cl \
 # RUN: | FileCheck --check-prefix=CHECK-FILTER %s
 
 # RUN: env LIT_XFAIL='false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN:   LIT_XFAIL_NOT='true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
 # RUN: %{lit} %{inputs}/xfail-cl \
 # RUN: | FileCheck --check-prefix=CHECK-FILTER %s
 
+# Check that --xfail and LIT_XFAIL_NOT always have precedence.
+
+# RUN: env LIT_XFAIL=true-xfail.txt \
+# RUN: %{lit} --xfail true-xfail.txt --xfail-not true-xfail.txt \
+# RUN:   --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \
+# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s
+
+# RUN: env LIT_XFAIL_NOT=true-xfail.txt LIT_XFAIL=true-xfail.txt \
+# RUN: %{lit} --xfail true-xfail.txt %{inputs}/xfail-cl/true-xfail.txt \
+# RUN: | FileCheck --check-prefix=CHECK-OVERRIDE %s
+
 # END.
 
-# CHECK-FILTER: Testing: 7 tests, {{[1-7]}} workers
+# CHECK-FILTER: Testing: 10 tests, {{[0-9]*}} workers
 # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: a :: false.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: false.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: false2.txt
 # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true.txt
+# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: true-xfail.txt
+# CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test-xfail.txt
+# CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test-xfail.txt
+
+# CHECK-OVERRIDE: Testing: 1 tests, {{[0-9]*}} workers
+# CHECK-OVERRIDE: {{^}}PASS: top-level-suite :: true-xfail.txt