Use Rocky Linux 8 for building Python packages (#4842)

* WIP: Use Rocky Linux 8 for building Python packages

* Remove obsolete config

* Use custom minimal OpenCASCADE build

* Update CI

* Update CI

* Update CI

* Revert excess changes

* Add comment

* Add comment

* Add readme

Author: oitel

.github/workflows/build-test-distribute.yml

@@ -332,7 +332,7 @@ jobs:
     if: ${{ needs.config.outputs.test_pip_build == 'true' }}
     uses: ./.github/workflows/pip-build.yml
     with:
-      docker_image_tag: ${{ needs.config.outputs.docker_image_tag }}
+      vcpkg_docker_image_tag: ${{ needs.config.outputs.vcpkg_docker_image_tag }}
 
   collect-stats:
     timeout-minutes: 5

.github/workflows/build-test-linux-vcpkg.yml

@@ -161,20 +161,6 @@ jobs:
           CXX: ${{ matrix.cxx-compiler }}
         run: make -f scripts/mrbind/generate.mk MODE=none -B --trace MESHLIB_SHLIB_DIR=build/${{matrix.config}}/bin CXX_FOR_BINDINGS=/usr/bin/clang++ DEPS_BASE_DIR=/opt/vcpkg/installed/${{ matrix.arch }}-linux-meshlib
 
-      # TODO: move to pip-build.yml after migrating from ubuntu20
-      #- name: Build Python shim libs
-      #  if: ${{ inputs.mrbind && !matrix.skip_mrbind }}
-      #  env:
-      #    CXX: ${{ matrix.cxx-compiler }}
-      #    CXX_FOR_BINDINGS: /usr/bin/clang++
-      #    VCPKG_BASE_DIR: /opt/vcpkg/installed/${{ matrix.arch }}-linux-meshlib
-      #  run: |
-      #    python3 -m pip install uv
-      #    for PYTHON_VER in $(cat scripts/mrbind-pybind11/python_versions.txt | xargs) ; do
-      #      export PKG_CONFIG_PATH=$(uv run -p ${PYTHON_VER} python -c "import sysconfig; print(sysconfig.get_config_var('LIBPC'))")
-      #      make shims -f scripts/mrbind/generate.mk -B --trace FOR_WHEEL=1 MESHLIB_SHLIB_DIR=build/${{matrix.config}}/bin PYTHON_VERSIONS=${PYTHON_VER}
-      #    done
-
       - name: Collect Timings
         run: ./scripts/devops/collect_timing_logs.sh linux-vcpkg-${{ matrix.arch }} ${{matrix.config}} "${{matrix.compiler}}"
 

.github/workflows/pip-build.yml

@@ -11,7 +11,7 @@ on:
   # This is the manual run from the UI.
   workflow_dispatch:
     inputs:
-      docker_image_tag:
+      vcpkg_docker_image_tag:
         # Empty string means the same thing as "latest", but we use an empty string here to ensure this works,
         #   because `types: [published]` can't have any inputs, so there we are forced to use an empty string, so it needs to work.
         default: ""
@@ -20,7 +20,7 @@ on:
   # This is the `uses:` call from other `.yml` files.
   workflow_call:
     inputs:
-      docker_image_tag:
+      vcpkg_docker_image_tag:
         required: true
         type: string
 
@@ -52,25 +52,23 @@ jobs:
     timeout-minutes: 80
     runs-on: ${{ matrix.runner }}
     container:
-      image: meshlib/meshlib-${{matrix.os}}:${{inputs.docker_image_tag || 'latest'}}
+      image: meshlib/meshlib-${{matrix.os}}:${{inputs.vcpkg_docker_image_tag || 'latest'}}
       options: ${{ matrix.container-options }}
     strategy:
       fail-fast: false
       matrix:
         platform: ["x86_64", "aarch64"]
         include:
           - platform: "x86_64"
-            os: "ubuntu20"
+            os: rockylinux8-vcpkg-x64
             container-options: "--user root"
             runner: ubuntu-latest
-            compiler: /usr/bin/clang++-11
+            compiler: /usr/bin/clang++
           - platform: "aarch64"
-            os: "ubuntu20-arm64"
+            os: rockylinux8-vcpkg-arm64
             container-options: " " # empty
             runner: [ self-hosted, linux-arm64, on-demand, meshinspector ]
-            # When using Clang 11 on Ubuntu Arm 20.04, we get some obscure ABI issues that fail some tests,
-            # see the comments in `test_regression/test_algorithms/test_decimate.py` for more details.
-            compiler: /usr/bin/clang++-18
+            compiler: /usr/bin/clang++
 
     steps:
       - name: Work-around possible permission issues
@@ -102,39 +100,50 @@ jobs:
           git config --global --add safe.directory ${GITHUB_WORKSPACE}/thirdparty/mrbind
           git submodule update --init --recursive --depth 1 thirdparty/imgui thirdparty/parallel-hashmap thirdparty/mrbind-pybind11 thirdparty/mrbind
 
-      - name: Python setup
-        env:
-          ALLOW_DELETING_EXISTING_PYTHON: 1
-        run: |
-          scripts/mrbind-pybind11/install_all_python_versions_ubuntu_reqs.sh
-
-      - name: Install thirdparty libs
+      - name: Install mrbind
         # Also print the amount of RAM. If there's not enough RAM, building MRBind bindings can fail. Not doing it in that step, because OOM fails can erase logs from the current step.
         run: |
-          ln -s /usr/local/lib/meshlib-thirdparty-lib/lib ./lib
-          ln -s /usr/local/lib/meshlib-thirdparty-lib/include ./include
-          ./scripts/mrbind/install_mrbind_ubuntu.sh
+          ./scripts/mrbind/install_mrbind_rockylinux.sh
           free -h
 
       - name: Build
         run: ./scripts/build_source.sh
         env:
           MESHLIB_BUILD_RELEASE: "ON"
           MESHLIB_BUILD_DEBUG: "OFF"
-          # When using Clang 11 on Ubuntu Arm 20.04, we get some obscure ABI issues that fail some tests,
-          # see the comments in `test_regression/test_algorithms/test_decimate.py` for more details.
-          CMAKE_CXX_COMPILER: ${{matrix.compiler}}
-          # FIXME: OpenGL libraries are included to wheel packages when the XDE module is used
-          # Enabling `-allow-unsupported-compiler` because we use Clang 18 on Arm, while Cuda supports at most 16.
+          CMAKE_CXX_COMPILER: ${{ matrix.compiler }}
           MR_CMAKE_OPTIONS: >
-            -DMRIOEXTRAS_OPENCASCADE_USE_XDE=OFF
             -DMRVIEWER_NO_GTK=ON
-            -DMESHLIB_BUILD_MRCUDA=ON
-            ${{ fromJSON('["", "-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"]')[matrix.platform == 'aarch64'] }}
+            -DMR_PCH_USE_EXTRA_HEADERS=ON
+            -DCMAKE_CUDA_HOST_COMPILER=/opt/rh/gcc-toolset-11/root/usr/bin/g++
 
       - name: Generate and build MRBind bindings
         run: |
-          make -f scripts/mrbind/generate.mk -B --trace FOR_WHEEL=1 CXX_FOR_ABI=${{matrix.compiler}}
+          make -f scripts/mrbind/generate.mk -B --trace \
+            FOR_WHEEL=1 \
+            CXX_FOR_ABI=${{ matrix.compiler }} \
+            CXX_FOR_BINDINGS=${{ matrix.compiler }} \
+            DEPS_BASE_DIR=${VCPKG_ROOT}/installed/${VCPKG_TRIPLET}
+
+      # install helper utility for Python version management
+      - name: Install uv
+        uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6
+
+      - name: Build Python shim libs
+        run: |
+          PYTHON_VERSIONS=$(cat scripts/mrbind-pybind11/python_versions.txt | xargs)
+          GET_SYSCONFIG_VAR=${PWD}/scripts/wheel/get_sysconfig_var.py
+          
+          for PYTHON_VER in ${PYTHON_VERSIONS} ; do
+            export PKG_CONFIG_PATH=$(uv run --python ${PYTHON_VER} ${GET_SYSCONFIG_VAR} LIBPC)
+          
+            make shims -f scripts/mrbind/generate.mk -B --trace \
+              FOR_WHEEL=1 \
+              CXX_FOR_ABI=${{ matrix.compiler }} \
+              CXX_FOR_BINDINGS=${{ matrix.compiler }} \
+              DEPS_BASE_DIR=${VCPKG_ROOT}/installed/${VCPKG_TRIPLET} \
+              PYTHON_VERSIONS=${PYTHON_VER}
+          done
 
       - name: Run Tests
         run: MR_LOCAL_RESOURCES=1 xvfb-run -a ./build/Release/bin/MeshViewer -hidden -noEventLoop -unloadPluginsAtEnd
@@ -143,14 +152,28 @@ jobs:
         run: build/Release/bin/MRTest
 
       - name: Python Tests
-        working-directory: ./build/Release/bin
-        run: python3 ./../../../scripts/run_python_test_script.py -multi-cmd
+        run: |
+          PYTHON_VERSIONS=$(cat scripts/mrbind-pybind11/python_versions.txt | xargs)
+          REQUIREMENTS_FILE=${PWD}/requirements/python.txt
+          GET_SYSCONFIG_VAR=${PWD}/scripts/wheel/get_sysconfig_var.py
+          
+          cd build/Release/bin
+          for PYTHON_VER in ${PYTHON_VERSIONS} ; do
+            PYTHON_LIB_DIR=$(uv run --python ${PYTHON_VER} ${GET_SYSCONFIG_VAR} LIBDIR)
+          
+            LD_LIBRARY_PATH=${PYTHON_LIB_DIR} \
+            uv run \
+              --python ${PYTHON_VER} \
+              --with-requirements ${REQUIREMENTS_FILE} \
+              --with pytest \
+              python3 ./../../../scripts/run_python_test_script.py -d '../test_python'
+          done
 
       - name: Create and fix Wheel
-        shell: bash
         run: |
           python3.12 -m venv ./wheel_venv
           source ./wheel_venv/bin/activate
+          python3.12 -m pip install patchelf
           python3.12 ./scripts/wheel/build_wheel.py --version ${{needs.setup.outputs.version_tag || '0'}}
 
       - name: Upload Python Stubs
@@ -407,7 +430,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: ["x86_64", "aarch64"]
-        os: ["ubuntu:20.04", "ubuntu:22.04", "debian:11-slim", "fedora:37", "fedora:39"]
+        py-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
         include:
           - platform: "x86_64"
             runner: ubuntu-latest
@@ -417,41 +440,44 @@ jobs:
             runner: [ self-hosted, linux-arm64, on-demand, meshinspector ]
             container-options: " " # empty
             container-prefix: "arm64v8/"
-          - os: "ubuntu:20.04"
-            py-version: "3.8"
-            py-cmd: "python3.8"
-          - os: "debian:11-slim"
-            py-version: "3.9"
-            py-cmd: "python3.9"
-          - os: "ubuntu:22.04"
-            py-version: "3.10"
-            py-cmd: "python3.10"
-          - os: "fedora:37"
-            py-version: "3.11"
-            py-cmd: "python3.11"
-          - os: "fedora:39"
-            py-version: "3.12"
-            py-cmd: "python3.12"
+          - py-version: "3.8"
+            os: "rockylinux:8"
+          - py-version: "3.9"
+            os: "debian:11-slim"
+          - py-version: "3.10"
+            os: "ubuntu:22.04"
+          - py-version: "3.11"
+            os: "fedora:37"
+          - py-version: "3.12"
+            os: "fedora:39"
+          - py-version: "3.13"
+            os: "fedora:42"
+    env:
+      PYTHON_CMD: python${{ matrix.py-version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{needs.setup.outputs.version_tag}}
 
-      - name: Ubuntu system setup
-        if: ${{ matrix.os == 'ubuntu:20.04' || matrix.os == 'ubuntu:22.04' || matrix.os == 'debian:11-slim' }}
-        run: apt -y update && apt -y upgrade && apt -y install curl libssl-dev python3-pip
+      - name: Rocky Linux system setup
+        if: ${{ matrix.os == 'rockylinux:8' }}
+        run: dnf -y install python38-pip
+
+      - name: Debian/Ubuntu system setup
+        if: ${{ matrix.os == 'debian:11-slim' || matrix.os == 'ubuntu:22.04' }}
+        run: apt -y update && apt -y install curl libssl-dev python3-pip
 
-      - name: Fedora 37 system setup
-        if: ${{matrix.os == 'fedora:37' || matrix.os == 'fedora:39'}}
-        run: dnf -y install python3 pip
+      - name: Fedora system setup
+        if: ${{ matrix.os == 'fedora:37' || matrix.os == 'fedora:39' || matrix.os == 'fedora:42' }}
+        run: dnf -y install python3-pip
 
       - name: Pip setup
         run: |
-          ${{ matrix.py-cmd }} -m pip install --upgrade pip
-          ${{ matrix.py-cmd }} -m pip uninstall -y meshlib
-          ${{ matrix.py-cmd }} -m pip install --upgrade -r ./requirements/python.txt
-          ${{ matrix.py-cmd }} -m pip install pytest
+          ${{ env.PYTHON_CMD }} -m pip install --upgrade pip
+          ${{ env.PYTHON_CMD }} -m pip uninstall -y meshlib
+          ${{ env.PYTHON_CMD }} -m pip install --upgrade -r ./requirements/python.txt
+          ${{ env.PYTHON_CMD }} -m pip install pytest
 
       - name: Download Meshlib wheel from Artifact
         uses: actions/download-artifact@v4
@@ -460,11 +486,11 @@ jobs:
           merge-multiple: true
 
       - name: Meshlib wheel install
-        run: ${{ matrix.py-cmd }} -m pip install ${{ matrix.pip-options }} ./meshlib-*.whl
+        run: ${{ env.PYTHON_CMD }} -m pip install ${{ matrix.pip-options }} ./meshlib-*.whl
 
       - name: Run Python tests
         working-directory: test_python
-        run: ${{ matrix.py-cmd }} -m pytest -s -v
+        run: ${{ env.PYTHON_CMD }} -m pytest -s -v
 
   windows-pip-test:
     needs: [windows-pip-build]

requirements/vcpkg-linux.txt

@@ -26,7 +26,7 @@ libe57format
 libharu
 libjpeg-turbo
 libzip
-opencascade[core]
+opencascade-minimal
 openctm
 openvdb
 parallel-hashmap

scripts/build_wheel.sh

@@ -111,7 +111,7 @@ def build_wheel():
 
     if SYSTEM == "Linux":
         # see also: https://github.com/mayeut/pep600_compliance
-        manylinux_version = "2_31"
+        manylinux_version = "2_28"
 
         os.chdir(WHEEL_ROOT_DIR)
         subprocess.check_call(

scripts/wheel/build_wheel.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""
+Helper script for retrieving non-system Python distribution info (libpython3 path, pkg-config path, etc.)
+"""
+import sysconfig
+
+if __name__ == '__main__':
+    import sys
+    print(sysconfig.get_config_var(sys.argv[1]))

scripts/wheel/get_sysconfig_var.py

@@ -117,9 +117,9 @@ IF(NOT MRIOEXTRAS_NO_STEP)
     ApplicationFramework
     DataExchange
   )
-  set(OpenCASCADE_REQUIRED_LIBRARIES TKernel TKBRep TKMath TKMesh TKDESTEP TKSTEP TKXSBase)
+  set(OpenCASCADE_REQUIRED_LIBRARIES TKernel TKBRep TKMath TKMesh TKDESTEP TKSTEP TKTopAlgo TKXSBase)
   IF(MRIOEXTRAS_OPENCASCADE_USE_XDE)
-    list(APPEND OpenCASCADE_REQUIRED_LIBRARIES TKXDESTEP TKLCAF TKXCAF TKTopAlgo)
+    list(APPEND OpenCASCADE_REQUIRED_LIBRARIES TKXDESTEP TKLCAF TKXCAF)
   ENDIF()
   FOREACH(LIBRARY_NAME ${OpenCASCADE_LIBRARIES})
     IF(${LIBRARY_NAME} IN_LIST OpenCASCADE_REQUIRED_LIBRARIES)

source/MRIOExtras/CMakeLists.txt

@@ -0,0 +1,24 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 17bc7f1638..c4283c3c4d 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -473,6 +473,9 @@ OCCT_IS_PRODUCT_REQUIRED (CSF_EIGEN CAN_USE_EIGEN)
+ 
+ set (OCCT_3RDPARTY_CMAKE_LIST)
+ 
++# define CSF variable
++OCCT_INCLUDE_CMAKE_FILE ("adm/cmake/occt_csf")
++
+ # Tcl (mandatory for Draw Harness)
+ if (USE_TCL)
+   message (STATUS "Info: TCL is used by OCCT")
+@@ -769,9 +772,6 @@ file(COPY ${CMAKE_SOURCE_DIR}/.clang-format DESTINATION ${CMAKE_SOURCE_DIR})
+ OCCT_MAKE_OS_WITH_BITNESS()
+ OCCT_MAKE_COMPILER_SHORT_NAME()
+ 
+-# define CSF variable
+-OCCT_INCLUDE_CMAKE_FILE ("adm/cmake/occt_csf")
+-
+ # do not define INSTALL_DIR_BIN for win.
+ # Leave library structure for win: <prefix>/win64/vc10/bin(d)
+ if (NOT DEFINED INSTALL_DIR_BIN)