runner: fix tracebacks for failed collectors getting longer and longer

Refs pytest-dev#12204 (comment)

changelog/12204.bugfix.rst

@@ -1,4 +1,7 @@
 Fix a regression in pytest 8.0 where tracebacks get longer and longer when multiple tests fail due to a shared higher-scope fixture which raised.
 
+Also fix a similar regression in pytest 5.4 for collectors which raise during setup.
+
 The fix necessitated internal changes which may affect some plugins:
 - ``FixtureDef.cached_result[2]`` is now a tuple ``(exc, tb)`` instead of ``exc``.
+- ``SetupState.stack`` failures are now a tuple ``(exc, tb)`` instead of ``exc``.

src/_pytest/runner.py

@@ -5,6 +5,7 @@
 import dataclasses
 import os
 import sys
+import types
 from typing import Callable
 from typing import cast
 from typing import Dict
@@ -488,8 +489,13 @@ def __init__(self) -> None:
             Tuple[
                 # Node's finalizers.
                 List[Callable[[], object]],
-                # Node's exception, if its setup raised.
-                Optional[Union[OutcomeException, Exception]],
+                # Node's exception and original traceback, if its setup raised.
+                Optional[
+                    Tuple[
+                        Union[OutcomeException, Exception],
+                        Optional[types.TracebackType],
+                    ]
+                ],
             ],
         ] = {}
 
@@ -502,7 +508,7 @@ def setup(self, item: Item) -> None:
         for col, (finalizers, exc) in self.stack.items():
             assert col in needed_collectors, "previous item was not torn down properly"
             if exc:
-                raise exc
+                raise exc[0].with_traceback(exc[1])
 
         for col in needed_collectors[len(self.stack) :]:
             assert col not in self.stack
@@ -511,7 +517,7 @@ def setup(self, item: Item) -> None:
             try:
                 col.setup()
             except TEST_OUTCOME as exc:
-                self.stack[col] = (self.stack[col][0], exc)
+                self.stack[col] = (self.stack[col][0], (exc, exc.__traceback__))
                 raise
 
     def addfinalizer(self, finalizer: Callable[[], object], node: Node) -> None:

testing/test_runner.py

@@ -142,6 +142,43 @@ def raiser(exc):
         assert isinstance(func.exceptions[0], TypeError)  # type: ignore
         assert isinstance(func.exceptions[1], ValueError)  # type: ignore
 
+    def test_cached_exception_doesnt_get_longer(self, pytester: Pytester) -> None:
+        """Regression test for #12204 (the "BTW" case)."""
+        pytester.makepyfile(test="")
+        # If the collector.setup() raises, all collected items error with this
+        # exception.
+        pytester.makeconftest(
+            """
+            import pytest
+
+            class MyItem(pytest.Item):
+                def runtest(self) -> None: pass
+
+            class MyBadCollector(pytest.Collector):
+                def collect(self):
+                    return [
+                        MyItem.from_parent(self, name="one"),
+                        MyItem.from_parent(self, name="two"),
+                        MyItem.from_parent(self, name="three"),
+                    ]
+
+                def setup(self):
+                    1 / 0
+
+            def pytest_collect_file(file_path, parent):
+                if file_path.name == "test.py":
+                    return MyBadCollector.from_parent(parent, name='bad')
+            """
+        )
+
+        result = pytester.runpytest_inprocess("--tb=native")
+        assert result.ret == ExitCode.TESTS_FAILED
+        failures = result.reprec.getfailures()  # type: ignore[attr-defined]
+        assert len(failures) == 3
+        lines1 = failures[1].longrepr.reprtraceback.reprentries[0].lines
+        lines2 = failures[2].longrepr.reprtraceback.reprentries[0].lines
+        assert len(lines1) == len(lines2)
+
 
 class BaseFunctionalTests:
     def test_passfunction(self, pytester: Pytester) -> None: