Merge pull request #1666 from Zac-HD/warn-collected-strategies

Warn on strategies collected as tests

Author: Zac-HD

hypothesis-python/RELEASE.rst

@@ -0,0 +1,8 @@
+RELEASE_TYPE: minor
+
+Our pytest plugin now warns you when strategy functions have been collected
+as tests, which may happen when e.g. using the
+:func:`@composite <hypothesis.strategies.composite>` decorator when you
+should be using ``@given(st.data())`` for inline draws.
+Such functions *always* pass when treated as tests, because the lazy creation
+of strategies mean that the function body is never actually executed!

hypothesis-python/src/hypothesis/extra/pytestplugin.py

@@ -22,6 +22,7 @@
 import pytest
 
 from hypothesis import Verbosity, core, settings
+from hypothesis._settings import note_deprecation
 from hypothesis.reporting import default as default_reporter
 from hypothesis.reporting import with_reporter
 from hypothesis.statistics import collector
@@ -183,8 +184,19 @@ def pytest_collection_modifyitems(items):
     for item in items:
         if not isinstance(item, pytest.Function):
             continue
-        if getattr(item.function, 'is_hypothesis_test', False):
+        if is_hypothesis_test(item.obj):
             item.add_marker('hypothesis')
+        if getattr(item.obj, 'is_hypothesis_strategy_function', False):
+            def note_strategy_is_not_test(*args, **kwargs):
+                note_deprecation(
+                    '%s is a function that returns a Hypothesis strategy, '
+                    'but pytest has collected it as a test function.  This '
+                    'is useless as the function body will never be executed.'
+                    'To define a test function, use @given instead of '
+                    '@composite.' % (item.nodeid,)
+                )
+
+            item.obj = note_strategy_is_not_test
 
 
 def load():

hypothesis-python/src/hypothesis/strategies.py

@@ -196,6 +196,8 @@ def accept(*args, **kwargs):
                 result.force_has_reusable_values = True
                 assert result.has_reusable_values
             return result
+
+        accept.is_hypothesis_strategy_function = True
         return accept
     return decorator
 

hypothesis-python/tests/pytest/test_capture.py

@@ -152,3 +152,20 @@ def test_healthcheck_traceback_is_hidden(testdir):
     def_line = get_line_num(def_token, result)
     timeout_line = get_line_num(timeout_token, result)
     assert timeout_line - def_line == 6
+
+
+COMPOSITE_IS_NOT_A_TEST = """
+from hypothesis.strategies import composite
+@composite
+def test_data_factory(draw):
+    assert False, 'Unreachable due to lazy construction'
+"""
+
+
+@pytest.mark.skipif(pytest.__version__[:3] == '3.0', reason='very very old')
+def test_deprecation_of_strategies_as_tests(testdir):
+    script = testdir.makepyfile(COMPOSITE_IS_NOT_A_TEST)
+    testdir.runpytest(script, '-Werror').assert_outcomes(failed=1)
+    result = testdir.runpytest(script)
+    result.assert_outcomes(passed=1)
+    result.stdout.fnmatch_lines(['*HypothesisDeprecationWarning*'])