gh-98903: Test suite fails with exit code 4 if no tests ran (#98904)

The Python test suite now fails wit exit code 4 if no tests ran. It
should help detecting typos in test names and test methods.

* Add "EXITCODE_" constants to Lib/test/libregrtest/main.py.
* Fix a typo: "NO TEST RUN" becomes "NO TESTS RAN"

Author: vstinner

Lib/test/libregrtest/main.py

@@ -28,6 +28,11 @@
 # Must be smaller than buildbot "1200 seconds without output" limit.
 EXIT_TIMEOUT = 120.0
 
+EXITCODE_BAD_TEST = 2
+EXITCODE_INTERRUPTED = 130
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+
 
 class Regrtest:
     """Execute a test suite.
@@ -493,15 +498,18 @@ def display_header(self):
         print("== encodings: locale=%s, FS=%s"
               % (locale.getencoding(), sys.getfilesystemencoding()))
 
+    def no_tests_run(self):
+        return not any((self.good, self.bad, self.skipped, self.interrupted,
+                        self.environment_changed))
+
     def get_tests_result(self):
         result = []
         if self.bad:
             result.append("FAILURE")
         elif self.ns.fail_env_changed and self.environment_changed:
             result.append("ENV CHANGED")
-        elif not any((self.good, self.bad, self.skipped, self.interrupted,
-            self.environment_changed)):
-            result.append("NO TEST RUN")
+        elif self.no_tests_run():
+            result.append("NO TESTS RAN")
 
         if self.interrupted:
             result.append("INTERRUPTED")
@@ -750,11 +758,13 @@ def _main(self, tests, kwargs):
         self.save_xml_result()
 
         if self.bad:
-            sys.exit(2)
+            sys.exit(EXITCODE_BAD_TEST)
         if self.interrupted:
-            sys.exit(130)
+            sys.exit(EXITCODE_INTERRUPTED)
         if self.ns.fail_env_changed and self.environment_changed:
-            sys.exit(3)
+            sys.exit(EXITCODE_ENV_CHANGED)
+        if self.no_tests_run():
+            sys.exit(EXITCODE_NO_TESTS_RAN)
         sys.exit(0)
 
 

Lib/test/test_regrtest.py

@@ -28,6 +28,11 @@
 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
 LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
 
+EXITCODE_BAD_TEST = 2
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+EXITCODE_INTERRUPTED = 130
+
 TEST_INTERRUPTED = textwrap.dedent("""
     from signal import SIGINT, raise_signal
     try:
@@ -497,7 +502,7 @@ def list_regex(line_format, tests):
             result.append('INTERRUPTED')
         if not any((good, result, failed, interrupted, skipped,
                     env_changed, fail_env_changed)):
-            result.append("NO TEST RUN")
+            result.append("NO TESTS RAN")
         elif not result:
             result.append('SUCCESS')
         result = ', '.join(result)
@@ -707,7 +712,7 @@ def test_failing(self):
         test_failing = self.create_test('failing', code=code)
         tests = [test_ok, test_failing]
 
-        output = self.run_tests(*tests, exitcode=2)
+        output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
         self.check_executed_tests(output, tests, failed=test_failing)
 
     def test_resources(self):
@@ -748,13 +753,14 @@ def test_random(self):
         test = self.create_test('random', code)
 
         # first run to get the output with the random seed
-        output = self.run_tests('-r', test)
+        output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
         randseed = self.parse_random_seed(output)
         match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
         test_random = int(match.group(1))
 
         # try to reproduce with the random seed
-        output = self.run_tests('-r', '--randseed=%s' % randseed, test)
+        output = self.run_tests('-r', '--randseed=%s' % randseed, test,
+                                exitcode=EXITCODE_NO_TESTS_RAN)
         randseed2 = self.parse_random_seed(output)
         self.assertEqual(randseed2, randseed)
 
@@ -813,7 +819,7 @@ def test_fromfile(self):
     def test_interrupted(self):
         code = TEST_INTERRUPTED
         test = self.create_test('sigint', code=code)
-        output = self.run_tests(test, exitcode=130)
+        output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
         self.check_executed_tests(output, test, omitted=test,
                                   interrupted=True)
 
@@ -838,7 +844,7 @@ def test_slowest_interrupted(self):
                     args = ("--slowest", "-j2", test)
                 else:
                     args = ("--slowest", test)
-                output = self.run_tests(*args, exitcode=130)
+                output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
                 self.check_executed_tests(output, test,
                                           omitted=test, interrupted=True)
 
@@ -878,7 +884,7 @@ def test_run(self):
                         builtins.__dict__['RUN'] = 1
         """)
         test = self.create_test('forever', code=code)
-        output = self.run_tests('--forever', test, exitcode=2)
+        output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
         self.check_executed_tests(output, [test]*3, failed=test)
 
     def check_leak(self, code, what):
@@ -887,7 +893,7 @@ def check_leak(self, code, what):
         filename = 'reflog.txt'
         self.addCleanup(os_helper.unlink, filename)
         output = self.run_tests('--huntrleaks', '3:3:', test,
-                                exitcode=2,
+                                exitcode=EXITCODE_BAD_TEST,
                                 stderr=subprocess.STDOUT)
         self.check_executed_tests(output, [test], failed=test)
 
@@ -969,7 +975,7 @@ def test_crashed(self):
         crash_test = self.create_test(name="crash", code=code)
 
         tests = [crash_test]
-        output = self.run_tests("-j2", *tests, exitcode=2)
+        output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
         self.check_executed_tests(output, tests, failed=crash_test,
                                   randomize=True)
 
@@ -1069,7 +1075,8 @@ def test_env_changed(self):
         self.check_executed_tests(output, [testname], env_changed=testname)
 
         # fail with --fail-env-changed
-        output = self.run_tests("--fail-env-changed", testname, exitcode=3)
+        output = self.run_tests("--fail-env-changed", testname,
+                                exitcode=EXITCODE_ENV_CHANGED)
         self.check_executed_tests(output, [testname], env_changed=testname,
                                   fail_env_changed=True)
 
@@ -1088,7 +1095,7 @@ def test_fail_always(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests("-w", testname, exitcode=2)
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
         self.check_executed_tests(output, [testname],
                                   failed=testname, rerun={testname: "test_fail_always"})
 
@@ -1123,7 +1130,8 @@ def test_bug(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0)
+        output = self.run_tests(testname, "-m", "nosuchtest",
+                                exitcode=EXITCODE_NO_TESTS_RAN)
         self.check_executed_tests(output, [testname], no_test_ran=testname)
 
     def test_no_tests_ran_skip(self):
@@ -1136,7 +1144,7 @@ def test_skipped(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests(testname, exitcode=0)
+        output = self.run_tests(testname)
         self.check_executed_tests(output, [testname])
 
     def test_no_tests_ran_multiple_tests_nonexistent(self):
@@ -1150,7 +1158,8 @@ def test_bug(self):
         testname = self.create_test(code=code)
         testname2 = self.create_test(code=code)
 
-        output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0)
+        output = self.run_tests(testname, testname2, "-m", "nosuchtest",
+                                exitcode=EXITCODE_NO_TESTS_RAN)
         self.check_executed_tests(output, [testname, testname2],
                                   no_test_ran=[testname, testname2])
 
@@ -1198,7 +1207,8 @@ def test_garbage(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests("--fail-env-changed", testname, exitcode=3)
+        output = self.run_tests("--fail-env-changed", testname,
+                                exitcode=EXITCODE_ENV_CHANGED)
         self.check_executed_tests(output, [testname],
                                   env_changed=[testname],
                                   fail_env_changed=True)
@@ -1224,7 +1234,8 @@ def test_sleep(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2)
+        output = self.run_tests("-j2", "--timeout=1.0", testname,
+                                exitcode=EXITCODE_BAD_TEST)
         self.check_executed_tests(output, [testname],
                                   failed=testname)
         self.assertRegex(output,
@@ -1256,7 +1267,8 @@ def test_unraisable_exc(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+        output = self.run_tests("--fail-env-changed", "-v", testname,
+                                exitcode=EXITCODE_ENV_CHANGED)
         self.check_executed_tests(output, [testname],
                                   env_changed=[testname],
                                   fail_env_changed=True)
@@ -1287,7 +1299,8 @@ def test_threading_excepthook(self):
         """)
         testname = self.create_test(code=code)
 
-        output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+        output = self.run_tests("--fail-env-changed", "-v", testname,
+                                exitcode=EXITCODE_ENV_CHANGED)
         self.check_executed_tests(output, [testname],
                                   env_changed=[testname],
                                   fail_env_changed=True)
@@ -1328,7 +1341,7 @@ def test_print_warning(self):
         for option in ("-v", "-W"):
             with self.subTest(option=option):
                 cmd = ["--fail-env-changed", option, testname]
-                output = self.run_tests(*cmd, exitcode=3)
+                output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
                 self.check_executed_tests(output, [testname],
                                           env_changed=[testname],
                                           fail_env_changed=True)
@@ -1373,7 +1386,8 @@ def test_leak_tmp_file(self):
         """)
         testnames = [self.create_test(code=code) for _ in range(3)]
 
-        output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
+        output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
+                                exitcode=EXITCODE_ENV_CHANGED)
         self.check_executed_tests(output, testnames,
                                   env_changed=testnames,
                                   fail_env_changed=True,

Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst

@@ -0,0 +1,2 @@
+The Python test suite now fails wit exit code 4 if no tests ran. It should
+help detecting typos in test names and test methods.