ENH: delegate computing wheel tags to the packaging module

Use the wheel contents only to determine whether the wheel contains
python ABI dependent modules or other platform dependent code.

Fixes mesonbuild#142, fixes mesonbuild#189, fixes mesonbuild#190.

Author: dnicolodi

meson.build

@@ -11,7 +11,6 @@ py.install_sources(
   'mesonpy/__init__.py',
   'mesonpy/_compat.py',
   'mesonpy/_elf.py',
-  'mesonpy/_tags.py',
   'mesonpy/_util.py',
   subdir: 'mesonpy',
 )

mesonpy/__init__.py

@@ -13,6 +13,7 @@
 import collections
 import contextlib
 import functools
+import importlib.machinery
 import io
 import itertools
 import json
@@ -35,6 +36,8 @@
     Union
 )
 
+import packaging.tags
+
 
 if sys.version_info < (3, 11):
     import tomli as tomllib
@@ -43,10 +46,9 @@
 
 import mesonpy._compat
 import mesonpy._elf
-import mesonpy._tags
 import mesonpy._util
 
-from mesonpy._compat import Collection, Iterator, Mapping, Path
+from mesonpy._compat import Iterator, Path
 
 
 if typing.TYPE_CHECKING:  # pragma: no cover
@@ -102,9 +104,33 @@ def _init_colors() -> Dict[str, str]:
 _STYLES = _init_colors()  # holds the color values, should be _COLORS or _NO_COLORS
 
 
-_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.so$')
-_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.pyd$')
-_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P<abi_number>[0-9]+)$')
+_EXTENSION_SUFFIXES = frozenset(importlib.machinery.EXTENSION_SUFFIXES)
+_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P<abi>[^.]+)\.)?(?:so|pyd)$')
+assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES)
+
+
+def _adjust_manylinux_tag(platform: str) -> str:
+    # The packaging module generates overly specific platforms tags on
+    # Linux.  The platforms tags on Linux evolved over time.  Relax
+    # the platform tags to maintain compatibility with old wheel
+    # installation tools.  The relaxed platform tags match the ones
+    # generated by the wheel package.
+    # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
+    return re.sub(r'^manylinux(1|2010|2014|_\d+_\d+)_(.*)$', r'linux_\2', platform)
+
+
+def _adjust_darwin_tag(platform: str) -> str:
+    # Override the macOS version if one is provided via the
+    # MACOS_DEPLOYMENT_TARGET environment variable.  Return it
+    # unchanged otherwise.
+    try:
+        version = tuple(map(int, os.environ.get('MACOS_DEPLOYMENT_TARGET', '').split('.')))[:2]
+    except ValueError:
+        version = None
+    if version is not None:
+        # str() to silence mypy
+        platform = str(next(packaging.tags.mac_platforms(version)))
+    return platform
 
 
 def _showwarning(
@@ -179,6 +205,11 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]:
     def _has_internal_libs(self) -> bool:
         return bool(self._wheel_files['mesonpy-libs'])
 
+    @property
+    def _has_extension_modules(self) -> bool:
+        # Assume that all code installed in {platlib} is Python ABI dependent.
+        return bool(self._wheel_files['platlib'])
+
     @property
     def basename(self) -> str:
         """Normalized wheel name and version (eg. meson_python-1.0.0)."""
@@ -187,14 +218,34 @@ def basename(self) -> str:
             version=self._project.version,
         )
 
+    @property
+    def tag(self) -> packaging.tags.Tag:
+        """Wheel tags."""
+        if self.is_pure:
+            return packaging.tags.Tag('py3', 'none', 'any')
+        # Get the most specific tag for the Python interpreter.
+        tag = next(packaging.tags.sys_tags())
+        if tag.platform.startswith('manylinux'):
+            tag = packaging.tags.Tag(tag.interpreter, tag.abi, _adjust_manylinux_tag(tag.platform))
+        elif tag.platform.startswith('darwin'):
+            tag = packaging.tags.Tag(tag.interpreter, tag.abi, _adjust_darwin_tag(tag.platform))
+        if not self._has_extension_modules:
+            # The wheel has platform dependent code (is not pure) but
+            # does not contain any extension module (does not
+            # distribute any file in {platlib}) thus use generic
+            # implementation and ABI tags.
+            return packaging.tags.Tag('py3', 'none', tag.platform)
+        if self._stable_abi:
+            # All distributed extension modules use the stable ABI.
+            return packaging.tags.Tag(tag.interpreter, self._stable_abi, tag.platform)
+        return tag
+
     @property
     def name(self) -> str:
-        """Wheel name, this includes the basename and tags."""
-        return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format(
+        """Wheel name, this includes the basename and tag."""
+        return '{basename}-{tag}'.format(
             basename=self.basename,
-            python_tag=self.python_tag,
-            abi_tag=self.abi_tag,
-            platform_tag=self.platform_tag,
+            tag=self.tag,
         )
 
     @property
@@ -226,10 +277,10 @@ def wheel(self) -> bytes:  # noqa: F811
             Wheel-Version: 1.0
             Generator: meson
             Root-Is-Purelib: {is_purelib}
-            Tag: {tags}
+            Tag: {tag}
         ''').strip().format(
             is_purelib='true' if self.is_pure else 'false',
-            tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}',
+            tag=self.tag,
         ).encode()
 
     @property
@@ -267,166 +318,40 @@ def _debian_python(self) -> bool:
         except ModuleNotFoundError:
             return False
 
-    @property
-    def python_tag(self) -> str:
-        selected_tag = self._select_abi_tag()
-        if selected_tag and selected_tag.python:
-            return selected_tag.python
-        return 'py3'
-
-    @property
-    def abi_tag(self) -> str:
-        selected_tag = self._select_abi_tag()
-        if selected_tag:
-            return selected_tag.abi
-        return 'none'
-
     @cached_property
-    def platform_tag(self) -> str:
-        if self.is_pure:
-            return 'any'
-        # XXX: Choose the sysconfig platform here and let something like auditwheel
-        #      fix it later if there are system dependencies (eg. replace it with a manylinux tag)
-        platform_ = sysconfig.get_platform()
-        parts = platform_.split('-')
-        if parts[0] == 'macosx':
-            target = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
-            if target:
-                print(
-                    '{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the '
-                    'platform tag to {target}{reset}'.format(target=target, **_STYLES)
-                )
-                parts[1] = target
-            else:
-                # If no target macOS version is specified fallback to
-                # platform.mac_ver() instead of sysconfig.get_platform() as the
-                # latter specifies the target macOS version Python was built
-                # against.
-                parts[1] = platform.mac_ver()[0]
-                if parts[1] >= '11':
-                    # Only pick up the major version, which changed from 10.X
-                    # to X.0 from macOS 11 onwards. See
-                    # https://github.com/mesonbuild/meson-python/issues/160
-                    parts[1] = parts[1].split('.')[0]
-
-            if parts[1] in ('11', '12'):
-                # Workaround for bug where pypa/packaging does not consider macOS
-                # tags without minor versions valid. Some Python flavors (Homebrew
-                # for example) on macOS started to do this in version 11, and
-                # pypa/packaging should handle things correctly from version 13 and
-                # forward, so we will add a 0 minor version to MacOS 11 and 12.
-                # https://github.com/mesonbuild/meson-python/issues/91
-                # https://github.com/pypa/packaging/issues/578
-                parts[1] += '.0'
-
-            platform_ = '-'.join(parts)
-        elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff:
-            # 32-bit Python running on an x86_64 host
-            # https://github.com/mesonbuild/meson-python/issues/123
-            parts[1] = 'i686'
-            platform_ = '-'.join(parts)
-        return platform_.replace('-', '_').replace('.', '_')
-
-    def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]:
-        """Try to calculate the Windows tag from the Python extension file name."""
-        match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename)
-        if not match:
-            return None
-        tag = match.group('tag')
+    def _stable_abi(self) -> Optional[str]:
+        """Determine stabe ABI compatibility.
 
-        try:
-            return mesonpy._tags.StableABITag(tag)
-        except ValueError:
-            return mesonpy._tags.InterpreterTag(tag)
-
-    def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]:
-        """Try to calculate the Posix tag from the Python extension file name."""
-        # sysconfig is not guaranted to export SHLIB_SUFFIX but let's be
-        # preventive and check its value to make sure it matches our expectations
-        try:
-            extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so')
-            if extension != '.so':
-                raise NotImplementedError(
-                    f"We don't currently support the {extension} extension. "
-                    'Please report this to https://github.com/mesonbuild/mesonpy/issues '
-                    'and include information about your operating system.'
-                )
-        except KeyError:
-            warnings.warn(
-                'sysconfig does not export SHLIB_SUFFIX, so we are unable to '
-                'perform the sanity check regarding the extension suffix. '
-                'Please report this to https://github.com/mesonbuild/mesonpy/issues '
-                'and include the output of `python -m sysconfig`.'
-            )
-        match = _LINUX_NATIVE_MODULE_REGEX.match(filename)
-        if not match:  # this file does not appear to be a native module
-            return None
-        tag = match.group('tag')
+        Examine all files installed in {platlib} that look like
+        extension modules (extension .pyd on Windows and .so on other
+        platforms) and, if they all share the same PEP 3149 filename
+        stable ABI tag, return it.
 
-        try:
-            return mesonpy._tags.StableABITag(tag)
-        except ValueError:
-            return mesonpy._tags.InterpreterTag(tag)
-
-    def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]:
-        """Try to calculate the ABI tag from the Python extension file name."""
-        if os.name == 'nt':
-            return self._calculate_file_abi_tag_heuristic_windows(filename)
-        # everything else *should* follow the POSIX way, at least to my knowledge
-        return self._calculate_file_abi_tag_heuristic_posix(filename)
-
-    def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str:
-        if len(files) > max_count:
-            files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))']
-        return ''.join(f'{prefix}- {file}\n' for file in files)
-
-    def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]:
-        """Map files into ABI tags."""
-        files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list)
-
-        for _, file in self._wheel_files['platlib']:
-            # if in platlib, calculate the ABI tag
-            tag = self._calculate_file_abi_tag_heuristic(file)
-            if tag:
-                files_by_tag[tag].append(file)
-
-        return files_by_tag
-
-    def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]:  # noqa: C901
-        """Given a list of ABI tags, selects the most specific one.
-
-        Raises an error if there are incompatible tags.
+        All files that look like extension modules are verified to
+        have a file name compatibel with what is expected by the
+        Python interpreter. An exception is raised otherwise.
+
+        Other files are ignored.
         """
-        # Possibilities:
-        #   - interpreter specific (cpython/pypy/etc, version)
-        #   - stable abi (abiX)
-        tags = self._files_by_tag()
-        selected_tag = None
-        for tag, files in tags.items():
-            # no selected tag yet, let's assign this one
-            if not selected_tag:
-                selected_tag = tag
-            # interpreter tag
-            elif isinstance(tag, mesonpy._tags.InterpreterTag):
-                if tag != selected_tag:
-                    if isinstance(selected_tag, mesonpy._tags.InterpreterTag):
-                        raise ValueError(
-                            'Found files with incompatible ABI tags:\n'
-                            + self._file_list_repr(tags[selected_tag])
-                            + '\tand\n'
-                            + self._file_list_repr(files)
-                        )
-                    selected_tag = tag
-            # stable ABI
-            elif isinstance(tag, mesonpy._tags.StableABITag):
-                if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag:
+        abis = []
+
+        for path, src in self._wheel_files['platlib']:
+            if os.name == 'nt' and path.suffix == '.pyd' or path.suffix == '.so':
+                match = re.match(r'^[^.]+(.*)$', path.name)
+                assert match is not None
+                suffix = match.group(1)
+                if suffix not in _EXTENSION_SUFFIXES:
                     raise ValueError(
-                        'Found files with incompatible ABI tags:\n'
-                        + self._file_list_repr(tags[selected_tag])
-                        + '\tand\n'
-                        + self._file_list_repr(files)
-                    )
-        return selected_tag
+                        f'Extension module {path!r} not compatible with Python interpreter. '
+                        f'Recognized extension suffixes are: {", ".join(repr(x) for x in _EXTENSION_SUFFIXES)}.')
+                match = _EXTENSION_SUFFIX_REGEX.match(suffix)
+                assert match is not None
+                abis.append(match.group('abi'))
+
+        stable = [x for x in abis if x and re.match(r'abi\d+', x)]
+        if len(stable) > 0 and len(stable) == len(abis) and all(x == stable[0] for x in stable[1:]):
+            return stable[0]
+        return None
 
     def _is_native(self, file: Union[str, pathlib.Path]) -> bool:
         """Check if file is a native file."""

mesonpy/_compat.py

@@ -10,11 +10,9 @@
 
 
 if sys.version_info >= (3, 9):
-    from collections.abc import (
-        Collection, Iterable, Iterator, Mapping, Sequence
-    )
+    from collections.abc import Collection, Iterable, Iterator, Sequence
 else:
-    from typing import Collection, Iterable, Iterator, Mapping, Sequence
+    from typing import Collection, Iterable, Iterator, Sequence
 
 
 if sys.version_info >= (3, 8):
@@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool:
     'Iterable',
     'Iterator',
     'Literal',
-    'Mapping',
     'Path',
     'Sequence',
 ]