ENH: determine wheel tags by Python interpreter introspection

The extension modules filename suffixes do not contain enough
information to correctly determine the wheel tags. Instead introspect
the Python interpreter to derive the wheel tags. This is the same
approach used by other PEP517 backends, most notably wheel. The wheel
contents only to determine whether the wheel contains python ABI
dependent modules or other platform dependent code.

The packaging module is the reference wheel tags derivation
implementation and it is used (or vendored) by most python packages
dealing with wheels. However, the API provided by packaging is
cumbersome to use for our purposes and, with the goal of merging this
code into Meson in the future, it is good to avoid an additional
dependency. Therefore, the tags derivation code is reimplemented.

Tests are added to verify that the tags produced by meson-python agree
with the ones produced by packaging to ensure that the two
implementations will not diverge.

Fixes mesonbuild#142, fixes mesonbuild#189, fixes mesonbuild#190.

Author: dnicolodi

mesonpy/__init__.py

@@ -13,6 +13,7 @@
 import collections
 import contextlib
 import functools
+import importlib.machinery
 import io
 import itertools
 import json
@@ -46,7 +47,7 @@
 import mesonpy._tags
 import mesonpy._util
 
-from mesonpy._compat import Collection, Iterator, Mapping, Path
+from mesonpy._compat import Iterator, Path
 
 
 if typing.TYPE_CHECKING:  # pragma: no cover
@@ -102,9 +103,9 @@ def _init_colors() -> Dict[str, str]:
 _STYLES = _init_colors()  # holds the color values, should be _COLORS or _NO_COLORS
 
 
-_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.so$')
-_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.pyd$')
-_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P<abi_number>[0-9]+)$')
+_EXTENSION_SUFFIXES = frozenset(importlib.machinery.EXTENSION_SUFFIXES)
+_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P<abi>[^.]+)\.)?(?:so|pyd)$')
+assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES)
 
 
 def _showwarning(
@@ -179,6 +180,11 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]:
     def _has_internal_libs(self) -> bool:
         return bool(self._wheel_files['mesonpy-libs'])
 
+    @property
+    def _has_extension_modules(self) -> bool:
+        # Assume that all code installed in {platlib} is Python ABI dependent.
+        return bool(self._wheel_files['platlib'])
+
     @property
     def basename(self) -> str:
         """Normalized wheel name and version (eg. meson_python-1.0.0)."""
@@ -187,14 +193,25 @@ def basename(self) -> str:
             version=self._project.version,
         )
 
+    @property
+    def tag(self) -> mesonpy._tags.Tag:
+        """Wheel tags."""
+        if self.is_pure:
+            return mesonpy._tags.Tag('py3', 'none', 'any')
+        if not self._has_extension_modules:
+            # The wheel has platform dependent code (is not pure) but
+            # does not contain any extension module (does not
+            # distribute any file in {platlib}) thus use generic
+            # implementation and ABI tags.
+            return mesonpy._tags.Tag('py3', 'none', None)
+        return mesonpy._tags.Tag(None, self._stable_abi, None)
+
     @property
     def name(self) -> str:
-        """Wheel name, this includes the basename and tags."""
-        return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format(
+        """Wheel name, this includes the basename and tag."""
+        return '{basename}-{tag}'.format(
             basename=self.basename,
-            python_tag=self.python_tag,
-            abi_tag=self.abi_tag,
-            platform_tag=self.platform_tag,
+            tag=self.tag,
         )
 
     @property
@@ -226,10 +243,10 @@ def wheel(self) -> bytes:  # noqa: F811
             Wheel-Version: 1.0
             Generator: meson
             Root-Is-Purelib: {is_purelib}
-            Tag: {tags}
+            Tag: {tag}
         ''').strip().format(
             is_purelib='true' if self.is_pure else 'false',
-            tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}',
+            tag=self.tag,
         ).encode()
 
     @property
@@ -267,166 +284,40 @@ def _debian_python(self) -> bool:
         except ModuleNotFoundError:
             return False
 
-    @property
-    def python_tag(self) -> str:
-        selected_tag = self._select_abi_tag()
-        if selected_tag and selected_tag.python:
-            return selected_tag.python
-        return 'py3'
-
-    @property
-    def abi_tag(self) -> str:
-        selected_tag = self._select_abi_tag()
-        if selected_tag:
-            return selected_tag.abi
-        return 'none'
-
     @cached_property
-    def platform_tag(self) -> str:
-        if self.is_pure:
-            return 'any'
-        # XXX: Choose the sysconfig platform here and let something like auditwheel
-        #      fix it later if there are system dependencies (eg. replace it with a manylinux tag)
-        platform_ = sysconfig.get_platform()
-        parts = platform_.split('-')
-        if parts[0] == 'macosx':
-            target = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
-            if target:
-                print(
-                    '{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the '
-                    'platform tag to {target}{reset}'.format(target=target, **_STYLES)
-                )
-                parts[1] = target
-            else:
-                # If no target macOS version is specified fallback to
-                # platform.mac_ver() instead of sysconfig.get_platform() as the
-                # latter specifies the target macOS version Python was built
-                # against.
-                parts[1] = platform.mac_ver()[0]
-                if parts[1] >= '11':
-                    # Only pick up the major version, which changed from 10.X
-                    # to X.0 from macOS 11 onwards. See
-                    # https://github.com/mesonbuild/meson-python/issues/160
-                    parts[1] = parts[1].split('.')[0]
-
-            if parts[1] in ('11', '12'):
-                # Workaround for bug where pypa/packaging does not consider macOS
-                # tags without minor versions valid. Some Python flavors (Homebrew
-                # for example) on macOS started to do this in version 11, and
-                # pypa/packaging should handle things correctly from version 13 and
-                # forward, so we will add a 0 minor version to MacOS 11 and 12.
-                # https://github.com/mesonbuild/meson-python/issues/91
-                # https://github.com/pypa/packaging/issues/578
-                parts[1] += '.0'
-
-            platform_ = '-'.join(parts)
-        elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff:
-            # 32-bit Python running on an x86_64 host
-            # https://github.com/mesonbuild/meson-python/issues/123
-            parts[1] = 'i686'
-            platform_ = '-'.join(parts)
-        return platform_.replace('-', '_').replace('.', '_')
-
-    def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]:
-        """Try to calculate the Windows tag from the Python extension file name."""
-        match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename)
-        if not match:
-            return None
-        tag = match.group('tag')
+    def _stable_abi(self) -> Optional[str]:
+        """Determine stabe ABI compatibility.
 
-        try:
-            return mesonpy._tags.StableABITag(tag)
-        except ValueError:
-            return mesonpy._tags.InterpreterTag(tag)
-
-    def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]:
-        """Try to calculate the Posix tag from the Python extension file name."""
-        # sysconfig is not guaranted to export SHLIB_SUFFIX but let's be
-        # preventive and check its value to make sure it matches our expectations
-        try:
-            extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so')
-            if extension != '.so':
-                raise NotImplementedError(
-                    f"We don't currently support the {extension} extension. "
-                    'Please report this to https://github.com/mesonbuild/mesonpy/issues '
-                    'and include information about your operating system.'
-                )
-        except KeyError:
-            warnings.warn(
-                'sysconfig does not export SHLIB_SUFFIX, so we are unable to '
-                'perform the sanity check regarding the extension suffix. '
-                'Please report this to https://github.com/mesonbuild/mesonpy/issues '
-                'and include the output of `python -m sysconfig`.'
-            )
-        match = _LINUX_NATIVE_MODULE_REGEX.match(filename)
-        if not match:  # this file does not appear to be a native module
-            return None
-        tag = match.group('tag')
+        Examine all files installed in {platlib} that look like
+        extension modules (extension .pyd on Windows and .so on other
+        platforms) and, if they all share the same PEP 3149 filename
+        stable ABI tag, return it.
 
-        try:
-            return mesonpy._tags.StableABITag(tag)
-        except ValueError:
-            return mesonpy._tags.InterpreterTag(tag)
-
-    def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]:
-        """Try to calculate the ABI tag from the Python extension file name."""
-        if os.name == 'nt':
-            return self._calculate_file_abi_tag_heuristic_windows(filename)
-        # everything else *should* follow the POSIX way, at least to my knowledge
-        return self._calculate_file_abi_tag_heuristic_posix(filename)
-
-    def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str:
-        if len(files) > max_count:
-            files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))']
-        return ''.join(f'{prefix}- {file}\n' for file in files)
-
-    def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]:
-        """Map files into ABI tags."""
-        files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list)
-
-        for _, file in self._wheel_files['platlib']:
-            # if in platlib, calculate the ABI tag
-            tag = self._calculate_file_abi_tag_heuristic(file)
-            if tag:
-                files_by_tag[tag].append(file)
-
-        return files_by_tag
-
-    def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]:  # noqa: C901
-        """Given a list of ABI tags, selects the most specific one.
-
-        Raises an error if there are incompatible tags.
+        All files that look like extension modules are verified to
+        have a file name compatibel with what is expected by the
+        Python interpreter. An exception is raised otherwise.
+
+        Other files are ignored.
         """
-        # Possibilities:
-        #   - interpreter specific (cpython/pypy/etc, version)
-        #   - stable abi (abiX)
-        tags = self._files_by_tag()
-        selected_tag = None
-        for tag, files in tags.items():
-            # no selected tag yet, let's assign this one
-            if not selected_tag:
-                selected_tag = tag
-            # interpreter tag
-            elif isinstance(tag, mesonpy._tags.InterpreterTag):
-                if tag != selected_tag:
-                    if isinstance(selected_tag, mesonpy._tags.InterpreterTag):
-                        raise ValueError(
-                            'Found files with incompatible ABI tags:\n'
-                            + self._file_list_repr(tags[selected_tag])
-                            + '\tand\n'
-                            + self._file_list_repr(files)
-                        )
-                    selected_tag = tag
-            # stable ABI
-            elif isinstance(tag, mesonpy._tags.StableABITag):
-                if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag:
+        abis = []
+
+        for path, src in self._wheel_files['platlib']:
+            if os.name == 'nt' and path.suffix == '.pyd' or path.suffix == '.so':
+                match = re.match(r'^[^.]+(.*)$', path.name)
+                assert match is not None
+                suffix = match.group(1)
+                if suffix not in _EXTENSION_SUFFIXES:
                     raise ValueError(
-                        'Found files with incompatible ABI tags:\n'
-                        + self._file_list_repr(tags[selected_tag])
-                        + '\tand\n'
-                        + self._file_list_repr(files)
-                    )
-        return selected_tag
+                        f'Extension module {str(path)!r} not compatible with Python interpreter. '
+                        f'Filename suffix {suffix!r} not in {set(_EXTENSION_SUFFIXES)}.')
+                match = _EXTENSION_SUFFIX_REGEX.match(suffix)
+                assert match is not None
+                abis.append(match.group('abi'))
+
+        stable = [x for x in abis if x and re.match(r'abi\d+', x)]
+        if len(stable) > 0 and len(stable) == len(abis) and all(x == stable[0] for x in stable[1:]):
+            return stable[0]
+        return None
 
     def _is_native(self, file: Union[str, pathlib.Path]) -> bool:
         """Check if file is a native file."""

mesonpy/_compat.py

@@ -10,11 +10,9 @@
 
 
 if sys.version_info >= (3, 9):
-    from collections.abc import (
-        Collection, Iterable, Iterator, Mapping, Sequence
-    )
+    from collections.abc import Collection, Iterable, Iterator, Sequence
 else:
-    from typing import Collection, Iterable, Iterator, Mapping, Sequence
+    from typing import Collection, Iterable, Iterator, Sequence
 
 
 if sys.version_info >= (3, 8):
@@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool:
     'Iterable',
     'Iterator',
     'Literal',
-    'Mapping',
     'Path',
     'Sequence',
 ]