Improve and complete, with parts from Zac-HD/hypothesis/plugin-checks

Author: jobh

hypothesis-python/RELEASE.rst

@@ -0,0 +1,6 @@
+RELEASE_TYPE: minor
+
+This release avoids creating a ``.hypothesis`` directory when using
+:func:`~hypothesis.strategies.register_type_strategy` (:issue:`3836`),
+and adds warnings for plugins which do so by other means or have
+other unintended side-effects.

hypothesis-python/src/_hypothesis_globals.py

@@ -0,0 +1,29 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+Module for globals shared between plugin(s) and the main hypothesis module, without
+depending on either. This file should have no imports outside of stdlib.
+"""
+
+import os
+
+
+in_initialization = 1
+"""If nonzero, indicates that hypothesis is still initializing (importing or loading
+the test environment). `import hypothesis` will cause this number to be decremented,
+and the pytest plugin increments at load time, then decrements it just before the test
+session starts. However, this leads to a hole in coverage if another pytest plugin
+imports hypothesis before our plugin is loaded. HYPOTHESIS_EXTEND_INITIALIZATION may
+be set to pre-increment the value on behalf of _hypothesis_pytestplugin, plugging the
+hole."""
+
+if os.environ.get("HYPOTHESIS_EXTEND_INITIALIZATION"):
+    in_initialization += 1

hypothesis-python/src/_hypothesis_pytestplugin.py

@@ -21,12 +21,15 @@
 
 import base64
 import json
+import os
 import sys
 import warnings
 from inspect import signature
 
 import pytest
 
+import _hypothesis_globals
+
 try:
     from _pytest.junitxml import xml_key
 except ImportError:
@@ -95,6 +98,19 @@ def __call__(self, msg):
     warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,), stacklevel=1)
 
 else:
+    # Restart side-effect detection as early as possible, to maximize coverage. We
+    # need balanced increment/decrement in configure/sessionstart to support nested
+    # pytest (e.g. runpytest_inprocess), so this early increment in effect replaces
+    # the first one in pytest_configure.
+    _configured = False
+    if not os.environ.get("HYPOTHESIS_EXTEND_INITIALIZATION"):
+        _hypothesis_globals.in_initialization += 1
+        if "hypothesis" in sys.modules:
+            # Some other plugin has imported hypothesis, so we'll check if there
+            # have been undetected side-effects and warn if so.
+            from hypothesis.configuration import notice_initialization_restarted
+
+            notice_initialization_restarted()
 
     def pytest_addoption(parser):
         group = parser.getgroup("hypothesis", "Hypothesis")
@@ -148,6 +164,12 @@ def pytest_report_header(config):
         return f"hypothesis profile {settings._current_profile!r}{settings_str}"
 
     def pytest_configure(config):
+        global _configured
+        # skip first increment because we pre-incremented at import time
+        if _configured:
+            _hypothesis_globals.in_initialization += 1
+        _configured = True
+
         config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.")
         if not _any_hypothesis_option(config):
             return
@@ -409,21 +431,7 @@ def pytest_collection_modifyitems(items):
                 item.add_marker("hypothesis")
 
     def pytest_sessionstart(session):
-        if "hypothesis" not in sys.modules:
-            return
-
-        from hypothesis.configuration import (
-            has_sideeffect_should_warn_been_called_after_import,
-        )
-        from hypothesis.errors import HypothesisSideeffectWarning
-
-        if has_sideeffect_should_warn_been_called_after_import():
-            warnings.warn(
-                "A plugin (or conftest.py) has caused hypothesis to perform undesired work during "
-                "initialization, possibly causing slowdown or creation of files. To pinpoint and explain "
-                "the problem, execute with environment 'PYTHONWARNINGS=error HYPOTHESIS_WARN_SIDEEFFECT=1'",
-                HypothesisSideeffectWarning,
-            )
+        _hypothesis_globals.in_initialization -= 1
 
     # Monkeypatch some internals to prevent applying @pytest.fixture() to a
     # function which has already been decorated with @hypothesis.given().

hypothesis-python/src/hypothesis/__init__.py

@@ -15,10 +15,7 @@
 failing examples it finds.
 """
 
-
-_is_importing = True  # noqa
-
-
+import _hypothesis_globals
 from hypothesis._settings import HealthCheck, Phase, Verbosity, settings
 from hypothesis.control import (
     assume,
@@ -58,4 +55,6 @@
 
 run()
 del run
-del _is_importing
+
+_hypothesis_globals.in_initialization -= 1
+del _hypothesis_globals

hypothesis-python/src/hypothesis/configuration.py

@@ -12,6 +12,7 @@
 import warnings
 from pathlib import Path
 
+import _hypothesis_globals
 import hypothesis
 from hypothesis.errors import HypothesisSideeffectWarning
 
@@ -26,14 +27,9 @@ def set_hypothesis_home_dir(directory):
 
 
 def storage_directory(*names, intent_to_write=True):
-    if intent_to_write and sideeffect_should_warn():
-        warnings.warn(
-            "Accessing the storage directory during import or initialization is "
-            "discouraged, as it may cause the .hypothesis directory to be created "
-            "even if hypothesis is not actually used. Typically, the fix will be "
-            "to defer initialization of strategies.",
-            HypothesisSideeffectWarning,
-            stacklevel=2,
+    if intent_to_write:
+        check_sideeffect_during_initialization(
+            f"accessing storage for {'/'.join(names)}"
         )
 
     global __hypothesis_home_directory
@@ -45,30 +41,43 @@ def storage_directory(*names, intent_to_write=True):
     return __hypothesis_home_directory.joinpath(*names)
 
 
-def _sideeffect_never_warn():
-    return False
-
-
-if os.environ.get("HYPOTHESIS_WARN_SIDEEFFECT"):
-
-    def sideeffect_should_warn():
-        return True
+_first_postinit_what = None
 
-else:
 
-    def sideeffect_should_warn():
-        if hasattr(hypothesis, "_is_importing"):
-            return True
-        else:
-            # We are no longer importing, patch this method to always return False from now on.
-            global sideeffect_should_warn
-            sideeffect_should_warn = _sideeffect_never_warn
-            return False
-
-
-def has_sideeffect_should_warn_been_called_after_import():
-    """We warn automatically if sideeffects are induced during import.
-    For sideeffects during initialization but after import, e.g. in pytest
-    plugins, this method can be used to show a catch-all warning at
-    start of session."""
-    return sideeffect_should_warn == _sideeffect_never_warn
+def check_sideeffect_during_initialization(what: str, extra: str = "") -> None:
+    """Called from locations that should not be executed during initialization, for example
+    touching disk or materializing lazy/deferred strategies from plugins. If initialization
+    is in progress, a warning is emitted."""
+    global _first_postinit_what
+    # This is not a particularly hot path, but neither is it doing productive work, so we want to
+    # minimize the cost by returning immediately. The drawback is that we require
+    # notice_initialization_restarted() to be called if in_initialization changes away from zero.
+    if _first_postinit_what is not None:
+        return
+    elif _hypothesis_globals.in_initialization:
+        # Note: -Werror is insufficient under pytest, as doesn't take effect until
+        # test session start.
+        warnings.warn(
+            f"Slow code in plugin: avoid {what} at import time!  Set PYTHONWARNINGS=error "
+            "to get a traceback and show which plugin is responsible." + extra,
+            HypothesisSideeffectWarning,
+            stacklevel=3,
+        )
+    else:
+        _first_postinit_what = what
+
+
+def notice_initialization_restarted(warn: bool = True) -> None:
+    """Reset _first_postinit_what, so that we don't think we're in post-init. Additionally, if it
+    was set that means that there has been a sideeffect that we haven't warned about, so do that
+    now (the warning text will be correct, and we also hint that the stacktrace can be improved).
+    """
+    global _first_postinit_what
+    if _first_postinit_what is not None:
+        what = _first_postinit_what
+        _first_postinit_what = None
+        if warn:
+            check_sideeffect_during_initialization(
+                what,
+                " Additionally, set HYPOTHESIS_EXTEND_INITIALIZATION=1 to pinpoint the exact location.",
+            )

hypothesis-python/src/hypothesis/errors.py

@@ -117,7 +117,7 @@ class HypothesisDeprecationWarning(HypothesisWarning, FutureWarning):
     """
 
 
-class HypothesisSideeffectWarning(HypothesisDeprecationWarning):
+class HypothesisSideeffectWarning(HypothesisWarning):
     """A warning issued by Hypothesis when it sees actions that are
     discouraged at import or initialization time because they have
     user-visible side effects.

hypothesis-python/src/hypothesis/strategies/_internal/deferred.py

@@ -10,6 +10,7 @@
 
 import inspect
 
+from hypothesis.configuration import check_sideeffect_during_initialization
 from hypothesis.errors import InvalidArgument
 from hypothesis.internal.reflection import get_pretty_function_description
 from hypothesis.strategies._internal.strategies import SearchStrategy, check_strategy
@@ -27,6 +28,8 @@ def __init__(self, definition):
     @property
     def wrapped_strategy(self):
         if self.__wrapped_strategy is None:
+            check_sideeffect_during_initialization(f"deferred evaluation of {self!r}")
+
             if not inspect.isfunction(self.__definition):
                 raise InvalidArgument(
                     f"Expected definition to be a function but got {self.__definition!r} "

hypothesis-python/src/hypothesis/strategies/_internal/lazy.py

@@ -13,7 +13,7 @@
 from typing import MutableMapping
 from weakref import WeakKeyDictionary
 
-from hypothesis.configuration import sideeffect_should_warn
+from hypothesis.configuration import check_sideeffect_during_initialization
 from hypothesis.errors import HypothesisSideeffectWarning
 from hypothesis.internal.reflection import (
     convert_keyword_arguments,
@@ -103,14 +103,7 @@ def calc_is_cacheable(self, recur):
     @property
     def wrapped_strategy(self):
         if self.__wrapped_strategy is None:
-            if sideeffect_should_warn():
-                warnings.warn(
-                    "Materializing lazy strategies at import or initialization time is "
-                    "discouraged, as it may cause a slowdown even when not actively "
-                    "using hypothesis.",
-                    HypothesisSideeffectWarning,
-                    stacklevel=2,
-                )
+            check_sideeffect_during_initialization(f"lazy evaluation of {self!r}")
 
             unwrapped_args = tuple(unwrap_strategies(s) for s in self.__args)
             unwrapped_kwargs = {

hypothesis-python/tests/cover/test_sideeffect_warnings.py

@@ -0,0 +1,52 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+
+from pathlib import Path
+import warnings
+
+import pytest
+
+import _hypothesis_globals
+from hypothesis import configuration as fs
+from hypothesis.errors import HypothesisSideeffectWarning
+from hypothesis import strategies as st
+
+IN_INITIALIZATION_ATTR = "in_initialization"
+
+
+@pytest.fixture
+def extend_initialization(monkeypatch):
+    monkeypatch.setattr(_hypothesis_globals, IN_INITIALIZATION_ATTR, 1)
+    fs.notice_initialization_restarted(warn=False)
+
+
+@pytest.mark.parametrize(
+    "sideeffect_script, warning_text",
+    [
+        ("st.integers().is_empty", "lazy evaluation"),
+        ("st.deferred(st.integers).is_empty", "deferred evaluation"),
+        ("fs.storage_directory()", "accessing storage"),
+    ],
+)
+def test_sideeffect_warning(sideeffect_script, warning_text, extend_initialization):
+    with pytest.warns(HypothesisSideeffectWarning, match=warning_text):
+        exec(sideeffect_script)
+
+
+def test_sideeffect_delayed_warning(monkeypatch, extend_initialization):
+    what = "synthetic side-effect"
+    # extend_initialization ensures we start at known clean slate (no delayed warnings).
+    # Then: stop initialization, check a side-effect operation, and restart it.
+    monkeypatch.setattr(_hypothesis_globals, IN_INITIALIZATION_ATTR, 0)
+    fs.check_sideeffect_during_initialization(what)
+    fs.check_sideeffect_during_initialization("ignored since not first")
+    with pytest.warns(HypothesisSideeffectWarning, match=what):
+        monkeypatch.setattr(_hypothesis_globals, IN_INITIALIZATION_ATTR, 1)
+        fs.notice_initialization_restarted()

hypothesis-python/tests/nocover/test_baseexception.py

@@ -8,12 +8,10 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-import warnings
-
 import pytest
 
 from hypothesis import given
-from hypothesis.errors import Flaky, HypothesisSideeffectWarning
+from hypothesis.errors import Flaky
 from hypothesis.strategies import composite, integers, none
 
 
@@ -127,11 +125,6 @@ def test_explanations(testdir, exc_name, use_composite):
         exception=exc_name, strategy="things()" if use_composite else "st.none()"
     )
     test_file = str(testdir.makepyfile(code))
-    with warnings.catch_warnings():
-        # running inprocess, side effects will be present from the beginning
-        warnings.simplefilter("ignore", HypothesisSideeffectWarning)
-        pytest_stdout = str(
-            testdir.runpytest_inprocess(test_file, "--tb=native").stdout
-        )
+    pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout)
     assert "x=101" in pytest_stdout
     assert exc_name in pytest_stdout

hypothesis-python/tests/nocover/test_scrutineer.py

@@ -9,11 +9,9 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import sys
-import warnings
 
 import pytest
 
-from hypothesis.errors import HypothesisSideeffectWarning
 from hypothesis.internal.compat import PYPY
 from hypothesis.internal.scrutineer import make_report
 
@@ -54,12 +52,7 @@ def get_reports(file_contents, *, testdir):
     # multi-line report strings which we expect to see in explain-mode output.
     # The list length is the number of explainable bugs, usually one.
     test_file = str(testdir.makepyfile(file_contents))
-    with warnings.catch_warnings():
-        # running inprocess, side effects will be present from the beginning
-        warnings.simplefilter("ignore", HypothesisSideeffectWarning)
-        pytest_stdout = str(
-            testdir.runpytest_inprocess(test_file, "--tb=native").stdout
-        )
+    pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout)
 
     explanations = {
         i: {(test_file, i)}