Merge pull request #5063 from asottile/importlib_metadata_v2

Switch to importlib-metadata

Author: asottile

changelog/5063.feature.rst

@@ -0,0 +1 @@
+Switch from ``pkg_resources`` to ``importlib-metadata`` for entrypoint detection for improved performance and import time.

setup.py

@@ -5,15 +5,16 @@
 INSTALL_REQUIRES = [
     "py>=1.5.0",
     "six>=1.10.0",
-    "setuptools",
+    "packaging",
     "attrs>=17.4.0",
     'more-itertools>=4.0.0,<6.0.0;python_version<="2.7"',
     'more-itertools>=4.0.0;python_version>"2.7"',
     "atomicwrites>=1.0",
     'funcsigs>=1.0;python_version<"3.0"',
     'pathlib2>=2.2.0;python_version<"3.6"',
     'colorama;sys_platform=="win32"',
-    "pluggy>=0.9,!=0.10,<1.0",
+    "pluggy>=0.12,<1.0",
+    "importlib-metadata>=0.12",
     "wcwidth",
 ]
 

src/_pytest/assertion/rewrite.py

@@ -64,7 +64,6 @@ def __init__(self, config):
         self.session = None
         self.modules = {}
         self._rewritten_names = set()
-        self._register_with_pkg_resources()
         self._must_rewrite = set()
         # flag to guard against trying to rewrite a pyc file while we are already writing another pyc file,
         # which might result in infinite recursion (#3506)
@@ -315,24 +314,6 @@ def is_package(self, name):
         tp = desc[2]
         return tp == imp.PKG_DIRECTORY
 
-    @classmethod
-    def _register_with_pkg_resources(cls):
-        """
-        Ensure package resources can be loaded from this loader. May be called
-        multiple times, as the operation is idempotent.
-        """
-        try:
-            import pkg_resources
-
-            # access an attribute in case a deferred importer is present
-            pkg_resources.__name__
-        except ImportError:
-            return
-
-        # Since pytest tests are always located in the file system, the
-        #  DefaultProvider is appropriate.
-        pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider)
-
     def get_data(self, pathname):
         """Optional PEP302 get_data API.
         """

src/_pytest/config/__init__.py

@@ -12,8 +12,10 @@
 import types
 import warnings
 
+import importlib_metadata
 import py
 import six
+from packaging.version import Version
 from pluggy import HookimplMarker
 from pluggy import HookspecMarker
 from pluggy import PluginManager
@@ -787,25 +789,17 @@ def _mark_plugins_for_rewrite(self, hook):
         modules or packages in the distribution package for
         all pytest plugins.
         """
-        import pkg_resources
-
         self.pluginmanager.rewrite_hook = hook
 
         if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
             # We don't autoload from setuptools entry points, no need to continue.
             return
 
-        # 'RECORD' available for plugins installed normally (pip install)
-        # 'SOURCES.txt' available for plugins installed in dev mode (pip install -e)
-        # for installed plugins 'SOURCES.txt' returns an empty list, and vice-versa
-        # so it shouldn't be an issue
-        metadata_files = "RECORD", "SOURCES.txt"
-
         package_files = (
-            entry.split(",")[0]
-            for entrypoint in pkg_resources.iter_entry_points("pytest11")
-            for metadata in metadata_files
-            for entry in entrypoint.dist._get_metadata(metadata)
+            str(file)
+            for dist in importlib_metadata.distributions()
+            if any(ep.group == "pytest11" for ep in dist.entry_points)
+            for file in dist.files
         )
 
         for name in _iter_rewritable_modules(package_files):
@@ -874,11 +868,10 @@ def _preparse(self, args, addopts=True):
 
     def _checkversion(self):
         import pytest
-        from pkg_resources import parse_version
 
         minver = self.inicfg.get("minversion", None)
         if minver:
-            if parse_version(minver) > parse_version(pytest.__version__):
+            if Version(minver) > Version(pytest.__version__):
                 raise pytest.UsageError(
                     "%s:%d: requires pytest-%s, actual pytest-%s'"
                     % (

src/_pytest/outcomes.py

@@ -8,6 +8,8 @@
 
 import sys
 
+from packaging.version import Version
+
 
 class OutcomeException(BaseException):
     """ OutcomeException and its subclass instances indicate and
@@ -175,15 +177,7 @@ def importorskip(modname, minversion=None, reason=None):
         return mod
     verattr = getattr(mod, "__version__", None)
     if minversion is not None:
-        try:
-            from pkg_resources import parse_version as pv
-        except ImportError:
-            raise Skipped(
-                "we have a required version for %r but can not import "
-                "pkg_resources to parse version strings." % (modname,),
-                allow_module_level=True,
-            )
-        if verattr is None or pv(verattr) < pv(minversion):
+        if verattr is None or Version(verattr) < Version(minversion):
             raise Skipped(
                 "module %r has __version__ %r, required is: %r"
                 % (modname, verattr, minversion),

testing/acceptance_test.py

@@ -9,6 +9,7 @@
 import types
 
 import attr
+import importlib_metadata
 import py
 import six
 
@@ -111,8 +112,6 @@ def test_option(pytestconfig):
 
     @pytest.mark.parametrize("load_cov_early", [True, False])
     def test_early_load_setuptools_name(self, testdir, monkeypatch, load_cov_early):
-        pkg_resources = pytest.importorskip("pkg_resources")
-
         testdir.makepyfile(mytestplugin1_module="")
         testdir.makepyfile(mytestplugin2_module="")
         testdir.makepyfile(mycov_module="")
@@ -124,38 +123,28 @@ def test_early_load_setuptools_name(self, testdir, monkeypatch, load_cov_early):
         class DummyEntryPoint(object):
             name = attr.ib()
             module = attr.ib()
-            version = "1.0"
-
-            @property
-            def project_name(self):
-                return self.name
+            group = "pytest11"
 
             def load(self):
                 __import__(self.module)
                 loaded.append(self.name)
                 return sys.modules[self.module]
 
-            @property
-            def dist(self):
-                return self
-
-            def _get_metadata(self, *args):
-                return []
-
         entry_points = [
             DummyEntryPoint("myplugin1", "mytestplugin1_module"),
             DummyEntryPoint("myplugin2", "mytestplugin2_module"),
             DummyEntryPoint("mycov", "mycov_module"),
         ]
 
-        def my_iter(group, name=None):
-            assert group == "pytest11"
-            for ep in entry_points:
-                if name is not None and ep.name != name:
-                    continue
-                yield ep
+        @attr.s
+        class DummyDist(object):
+            entry_points = attr.ib()
+            files = ()
+
+        def my_dists():
+            return (DummyDist(entry_points),)
 
-        monkeypatch.setattr(pkg_resources, "iter_entry_points", my_iter)
+        monkeypatch.setattr(importlib_metadata, "distributions", my_dists)
         params = ("-p", "mycov") if load_cov_early else ()
         testdir.runpytest_inprocess(*params)
         if load_cov_early:

testing/test_assertion.py

@@ -137,12 +137,12 @@ def test_foo(pytestconfig):
     def test_pytest_plugins_rewrite_module_names_correctly(self, testdir):
         """Test that we match files correctly when they are marked for rewriting (#2939)."""
         contents = {
-            "conftest.py": """
+            "conftest.py": """\
                 pytest_plugins = "ham"
             """,
             "ham.py": "",
             "hamster.py": "",
-            "test_foo.py": """
+            "test_foo.py": """\
                 def test_foo(pytestconfig):
                     assert pytestconfig.pluginmanager.rewrite_hook.find_module('ham') is not None
                     assert pytestconfig.pluginmanager.rewrite_hook.find_module('hamster') is None
@@ -153,14 +153,13 @@ def test_foo(pytestconfig):
         assert result.ret == 0
 
     @pytest.mark.parametrize("mode", ["plain", "rewrite"])
-    @pytest.mark.parametrize("plugin_state", ["development", "installed"])
-    def test_installed_plugin_rewrite(self, testdir, mode, plugin_state, monkeypatch):
+    def test_installed_plugin_rewrite(self, testdir, mode, monkeypatch):
         monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
         # Make sure the hook is installed early enough so that plugins
         # installed via setuptools are rewritten.
         testdir.tmpdir.join("hampkg").ensure(dir=1)
         contents = {
-            "hampkg/__init__.py": """
+            "hampkg/__init__.py": """\
                 import pytest
 
                 @pytest.fixture
@@ -169,7 +168,7 @@ def check(values, value):
                         assert values.pop(0) == value
                     return check
             """,
-            "spamplugin.py": """
+            "spamplugin.py": """\
             import pytest
             from hampkg import check_first2
 
@@ -179,46 +178,31 @@ def check(values, value):
                     assert values.pop(0) == value
                 return check
             """,
-            "mainwrapper.py": """
-            import pytest, pkg_resources
-
-            plugin_state = "{plugin_state}"
-
-            class DummyDistInfo(object):
-                project_name = 'spam'
-                version = '1.0'
-
-                def _get_metadata(self, name):
-                    # 'RECORD' meta-data only available in installed plugins
-                    if name == 'RECORD' and plugin_state == "installed":
-                        return ['spamplugin.py,sha256=abc,123',
-                                'hampkg/__init__.py,sha256=abc,123']
-                    # 'SOURCES.txt' meta-data only available for plugins in development mode
-                    elif name == 'SOURCES.txt' and plugin_state == "development":
-                        return ['spamplugin.py',
-                                'hampkg/__init__.py']
-                    return []
+            "mainwrapper.py": """\
+            import pytest, importlib_metadata
 
             class DummyEntryPoint(object):
                 name = 'spam'
                 module_name = 'spam.py'
-                attrs = ()
-                extras = None
-                dist = DummyDistInfo()
+                group = 'pytest11'
 
-                def load(self, require=True, *args, **kwargs):
+                def load(self):
                     import spamplugin
                     return spamplugin
 
-            def iter_entry_points(group, name=None):
-                yield DummyEntryPoint()
+            class DummyDistInfo(object):
+                version = '1.0'
+                files = ('spamplugin.py', 'hampkg/__init__.py')
+                entry_points = (DummyEntryPoint(),)
+                metadata = {'name': 'foo'}
 
-            pkg_resources.iter_entry_points = iter_entry_points
+            def distributions():
+                return (DummyDistInfo(),)
+
+            importlib_metadata.distributions = distributions
             pytest.main()
-            """.format(
-                plugin_state=plugin_state
-            ),
-            "test_foo.py": """
+            """,
+            "test_foo.py": """\
             def test(check_first):
                 check_first([10, 30], 30)