Add support for tests that fail at runtime (#418)

That is compilation must be successful, but running the test should
return a non-zero exit code.

This is useful for testing that code correctly asserts.

Author: ahendriksen

libcudacxx/.upstream-tests/test/lit.cfg

@@ -15,7 +15,7 @@ if 'PYLINT_IMPORT' in os.environ:
 config.name = 'libcu++'
 
 # suffixes: A list of file extensions to treat as test files.
-config.suffixes = ['.pass.cpp', '.fail.cpp', '.sh.cpp']
+config.suffixes = ['.pass.cpp', '.fail.cpp', '.runfail.cpp', '.sh.cpp']
 
 # test_source_root: The root path where tests are located.
 config.test_source_root = os.path.dirname(__file__)

libcudacxx/.upstream-tests/utils/libcudacxx/test/format.py

@@ -29,6 +29,8 @@ class LibcxxTestFormat(object):
       FOO.pass.cpp - Executable test which should compile, run, and exit with
                      code 0.
       FOO.fail.cpp - Negative test case which is expected to fail compilation.
+      FOO.runfail.cpp - Negative test case which is expected to compile, run,
+                        and exit with non-zero exit code.
       FOO.sh.cpp   - A test that uses LIT's ShTest format.
     """
 
@@ -88,6 +90,7 @@ def _execute(self, test, lit_config):
         is_sh_test = name_root.endswith('.sh')
         is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
         is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
+        is_runfail_test = name.endswith('.runfail.cpp') or name.endswith('.runfail.mm')
         is_objcxx_test = name.endswith('.mm')
         is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
                              name.endswith('.arc.fail.mm')
@@ -166,6 +169,10 @@ def _execute(self, test, lit_config):
         elif is_pass_test:
             return self._evaluate_pass_test(test, tmpBase, lit_config,
                                             test_cxx, parsers)
+        elif is_runfail_test:
+            return self._evaluate_pass_test(test, tmpBase, lit_config,
+                                            test_cxx, parsers,
+                                            run_should_pass=False)
         else:
             # No other test type is supported
             assert False
@@ -174,7 +181,7 @@ def _clean(self, exec_path):  # pylint: disable=no-self-use
         libcudacxx.util.cleanFile(exec_path)
 
     def _evaluate_pass_test(self, test, tmpBase, lit_config,
-                            test_cxx, parsers):
+                            test_cxx, parsers, run_should_pass=True):
         execDir = os.path.dirname(test.getExecPath())
         source_path = test.getSourcePath()
         exec_path = tmpBase + '.exe'
@@ -210,14 +217,18 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config,
                                                       env)
                 report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
                 report += libcudacxx.util.makeReport(cmd, out, err, rc)
-                if rc == 0:
+                result_expected = (rc == 0) == run_should_pass
+                if result_expected:
                     res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
                     return lit.Test.Result(res, report)
                 # Rarely devices are unavailable, so just restart the test to avoid false negatives.
                 elif rc != 0 and "cudaErrorDevicesUnavailable" in out and max_retry <= 5:
                     max_retry += 1
-                elif rc != 0 and retry_count + 1 >= max_retry:
-                    report += "Compiled test failed unexpectedly!"
+                elif retry_count + 1 == max_retry:
+                    if run_should_pass:
+                        report += "Compiled test failed unexpectedly!"
+                    else:
+                        report += "Compiled test succeeded unexpectedly!"
                     return lit.Test.Result(lit.Test.FAIL, report)
 
             assert False # Unreachable

libcudacxx/libcxx/utils/libcxx/test/format.py

@@ -29,6 +29,8 @@ class LibcxxTestFormat(object):
       FOO.pass.cpp - Executable test which should compile, run, and exit with
                      code 0.
       FOO.fail.cpp - Negative test case which is expected to fail compilation.
+      FOO.runfail.cpp - Negative test case which is expected to compile, run,
+                        and exit with non-zero exit code.
       FOO.sh.cpp   - A test that uses LIT's ShTest format.
     """
 
@@ -88,6 +90,7 @@ def _execute(self, test, lit_config):
         is_sh_test = name_root.endswith('.sh')
         is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
         is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
+        is_runfail_test = name.endswith('.runfail.cpp') or name.endswith('.runfail.mm')
         is_objcxx_test = name.endswith('.mm')
         is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
                              name.endswith('.arc.fail.mm')
@@ -163,15 +166,18 @@ def _execute(self, test, lit_config):
         elif is_pass_test:
             return self._evaluate_pass_test(test, tmpBase, lit_config,
                                             test_cxx, parsers)
+        elif is_runfail_test:
+            return self._evaluate_pass_test(test, tmpBase, lit_config,
+                                            test_cxx, parsers,
+                                            run_should_pass=False)
         else:
-            # No other test type is supported
-            assert False
+            assert False and "no other test" # No other test type is supported
 
     def _clean(self, exec_path):  # pylint: disable=no-self-use
         libcxx.util.cleanFile(exec_path)
 
     def _evaluate_pass_test(self, test, tmpBase, lit_config,
-                            test_cxx, parsers):
+                            test_cxx, parsers, run_should_pass=True):
         execDir = os.path.dirname(test.getExecPath())
         source_path = test.getSourcePath()
         exec_path = tmpBase + '.exe'
@@ -207,11 +213,15 @@ def _evaluate_pass_test(self, test, tmpBase, lit_config,
                                                       env)
                 report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
                 report += libcxx.util.makeReport(cmd, out, err, rc)
-                if rc == 0:
+                result_expected = (rc == 0) == run_should_pass
+                if result_expected:
                     res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
                     return lit.Test.Result(res, report)
-                elif rc != 0 and retry_count + 1 == max_retry:
-                    report += "Compiled test failed unexpectedly!"
+                elif retry_count + 1 == max_retry:
+                    if run_should_pass:
+                        report += "Compiled test failed unexpectedly!"
+                    else:
+                        report += "Compiled test succeeded unexpectedly!"
                     return lit.Test.Result(lit.Test.FAIL, report)
 
             assert False # Unreachable