Backport PR #5179 on branch yt-4.4.x (WHL: enable abi3 wheels on CPython 3.11 and newer)

.github/workflows/wheels.yaml

@@ -12,6 +12,8 @@ on:
       - '.github/workflows/wheels.yaml'
       - MANIFEST.in
       - pyproject.toml
+      - setup.py
+      - setupext.py
   workflow_dispatch:
 
 

doc/source/installing.rst

@@ -112,11 +112,23 @@ Note that all format names are normalized to lower case.
 Building from source
 ++++++++++++++++++++
 
-To build yt from source, you need ``git``, and a C compiler (such as ``gcc``
-or ``clang``).
-
-Then run
-
+There are a couple ways to build yt from source with e.g., ``pip``, all of which
+require a C compiler (such as ``gcc`` or ``clang``).
+
+yt is primarily distributed on PyPI, in the form of pre-built binaries (wheels).
+Since version 4.5.0, these binaries are optimized for portability accross Python versions.
+
+If you need a stable release, but pre-built binaries are not available for your platform,
+``pip install`` will automatically select a source distribution and compile the package
+for you. You may opt-into this behavior deliberately by specifying the ``--no-binary``
+flag, in which case the resulting installation might be slightly more performant, because
+it will be compiled specifically for your Python version.
+If, on the other hand, you *specifically* want a portable binary (as the ones we provide on
+PyPI), this is achieved by setting ``YT_LIMITED_API=1`` in your build environment.
+
+You may also want to build yt directly from the github repository (which requires ``git``),
+for instance if you need the latest development version, or if you want to contribute to
+the project. Run
 .. code-block:: bash
 
   $ git clone https://github.com/yt-project/yt

pyproject.toml

@@ -5,6 +5,7 @@ requires = [
   "Cython>=3.0.3",
   "numpy>=2.0.0",
   "ewah-bool-utils>=1.2.0",
+  "wheel>=0.38.0",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -470,6 +471,7 @@ test-extras = "test"
 test-command = [
     "python -m pytest -c {project}/pyproject.toml --rootdir . --color=yes --pyargs yt -ra",
 ]
+environment = {"YT_LIMITED_API" = "1"}
 
 [[tool.cibuildwheel.overrides]]
 # Install nightly wheels for matplotlib, not yet available on PyPI.

setup.py

@@ -83,7 +83,7 @@
 lib_exts += embree_libs
 
 # This overrides using lib_exts, so it has to happen after lib_exts is fully defined
-build_ext, sdist = create_build_ext(lib_exts, cythonize_aliases)
+build_ext, sdist, bdist_wheel = create_build_ext(lib_exts, cythonize_aliases)
 
 
 # Force setuptools to consider that there are ext modules, even if empty.
@@ -122,7 +122,7 @@ def has_ext_modules(self):
     )
 
     setup(
-        cmdclass={"sdist": sdist, "build_ext": build_ext},
+        cmdclass={"sdist": sdist, "build_ext": build_ext, "bdist_wheel": bdist_wheel},
         distclass=BinaryDistribution,
         libraries=[fixed_interp_lib],
         ext_modules=[],  # !!! We override this inside build_ext above

setupext.py

@@ -19,9 +19,19 @@
 from setuptools.command.sdist import sdist as _sdist
 from setuptools.errors import CompileError, LinkError
 import importlib.resources as importlib_resources
+from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
 
 log = logging.getLogger("setupext")
 
+USE_PY_LIMITED_API = (
+    os.getenv('YT_LIMITED_API', '0') == '1'
+    and sys.version_info >= (3, 11)
+    and not sysconfig.get_config_var("Py_GIL_DISABLED")
+)
+ABI3_TARGET_VERSION = "".join(str(_) for _ in sys.version_info[:2])
+ABI3_TARGET_HEX = hex(sys.hexversion & 0xFFFF00F0)
+
+
 @contextlib.contextmanager
 def stdchannel_redirected(stdchannel, dest_filename):
     """
@@ -433,6 +443,10 @@ def finalize_options(self):
             self.include_dirs.append(ewah_bool_utils.get_include())
 
             define_macros = NUMPY_MACROS
+            if USE_PY_LIMITED_API:
+                define_macros.append(("Py_LIMITED_API", ABI3_TARGET_HEX))
+                for ext in self.extensions:
+                    ext.py_limited_api = True
 
             if self.define is None:
                 self.define = define_macros
@@ -477,4 +491,13 @@ def run(self):
             )
             _sdist.run(self)
 
-    return build_ext, sdist
+    class bdist_wheel(_bdist_wheel):
+        def get_tag(self):
+            python, abi, plat = super().get_tag()
+
+            if python.startswith("cp") and USE_PY_LIMITED_API:
+                return f"cp{ABI3_TARGET_VERSION}", "abi3", plat
+
+            return python, abi, plat
+
+    return build_ext, sdist, bdist_wheel