refactor: consolidate modernization and test overhaul #758
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Build Configuration | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test_matrix: | |
| description: 'Test full matrix or subset' | |
| required: false | |
| default: 'subset' | |
| type: choice | |
| options: | |
| - 'full' | |
| - 'subset' | |
| python_version: | |
| description: 'Python version to build/test (single value; default 3.12)' | |
| required: false | |
| default: '3.12' | |
| type: choice | |
| options: | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - '3.13' | |
| - '3.14' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/test-build.yml' | |
| - '.github/workflows/pgo-validate.yml' | |
| - '.github/workflows/publish.yml' | |
| - 'pyproject.toml' | |
| - 'uv.lock' | |
| - 'tools/scripts/pgo_training.py' | |
| - 'sqlspec/**' | |
| - 'tools/scripts/mypyc_smoke.py' | |
| permissions: | |
| contents: read | |
| checks: read | |
| # Cancel in-flight runs of this workflow on the same ref when a newer | |
| # commit lands. Saves runner minutes during a churn of PR pushes. Not | |
| # applied to publish.yml — releases must never be cancelled mid-upload. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| detect-mode: | |
| name: Detect matrix mode | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix_mode: ${{ steps.detect.outputs.matrix_mode }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| # workflow_dispatch with test_matrix=full always wins. Otherwise, | |
| # promote a PR to the full per-OS x per-py matrix when its diff | |
| # touches anything that could affect wheel-build correctness | |
| # (workflow yaml, packaging metadata, PGO training, smoke script). | |
| # Non-infra PRs stay in subset mode (linux x py3.12) for fast feedback. | |
| - name: Detect matrix mode | |
| id: detect | |
| env: | |
| DISPATCH_MODE: ${{ github.event.inputs.test_matrix }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| shell: bash | |
| run: | | |
| if [ "$DISPATCH_MODE" = "full" ]; then | |
| mode=full | |
| reason="workflow_dispatch input" | |
| else | |
| mode=subset | |
| reason="default" | |
| if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ -n "$BASE_SHA" ] && [ -n "$HEAD_SHA" ]; then | |
| changed=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" || true) | |
| if printf '%s\n' "$changed" | grep -qE '^(\.github/workflows/.*\.yml|pyproject\.toml|uv\.lock|tools/scripts/(pgo_training|mypyc_smoke)\.py)$'; then | |
| mode=full | |
| reason="PR touches build-matrix infra" | |
| fi | |
| fi | |
| fi | |
| echo "matrix_mode=$mode" >> "$GITHUB_OUTPUT" | |
| echo "Resolved matrix_mode=$mode ($reason)" | |
| wait-for-cheap-ci: | |
| name: Wait for quality and unit checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Skip wait for manual runs | |
| if: github.event_name != 'pull_request' | |
| run: echo "workflow_dispatch builds do not wait for PR checks" | |
| - name: Wait for Tests And Linting checks | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| REPOSITORY: ${{ github.repository }} | |
| REQUIRED_CHECKS: | | |
| quality | |
| test-unit (3.10) | |
| test-unit (3.11) | |
| test-unit (3.12) | |
| test-unit (3.13) | |
| test-unit (3.14) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| deadline=$((SECONDS + 25 * 60)) | |
| while true; do | |
| response="$(gh api "repos/$REPOSITORY/commits/$HEAD_SHA/check-runs?per_page=100")" | |
| pending=0 | |
| while IFS= read -r check_name; do | |
| [ -z "$check_name" ] && continue | |
| row="$(jq -r --arg name "$check_name" ' | |
| [.check_runs[] | select(.name == $name)] | |
| | sort_by(.started_at // .created_at // "") | |
| | last | |
| | if . == null then "" else [.status, (.conclusion // ""), .html_url] | @tsv end | |
| ' <<< "$response")" | |
| if [ -z "$row" ]; then | |
| echo "Waiting for check to appear: $check_name" | |
| pending=$((pending + 1)) | |
| continue | |
| fi | |
| IFS=$'\t' read -r status conclusion url <<< "$row" | |
| case "$status:$conclusion" in | |
| completed:success) | |
| echo "Passed: $check_name" | |
| ;; | |
| completed:*) | |
| echo "::error::Required check '$check_name' completed with conclusion '$conclusion': $url" | |
| exit 1 | |
| ;; | |
| *) | |
| echo "Waiting for $check_name ($status): $url" | |
| pending=$((pending + 1)) | |
| ;; | |
| esac | |
| done <<< "$REQUIRED_CHECKS" | |
| if [ "$pending" -eq 0 ]; then | |
| echo "Quality and unit checks passed for $HEAD_SHA" | |
| exit 0 | |
| fi | |
| if [ "$SECONDS" -ge "$deadline" ]; then | |
| echo "::error::Timed out waiting for quality and unit checks for $HEAD_SHA" | |
| exit 1 | |
| fi | |
| sleep 30 | |
| done | |
| build-source: | |
| name: Build source distribution | |
| runs-on: ubuntu-latest | |
| needs: [wait-for-cheap-ci] | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| - name: Set up Python | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --extra performance --group build | |
| - name: Build source distribution | |
| run: uv build --sdist | |
| - name: Upload source artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: source-dist | |
| path: dist/*.tar.gz | |
| build-wheels-standard: | |
| name: Build standard pure Python wheel | |
| runs-on: ubuntu-latest | |
| needs: [wait-for-cheap-ci] | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| - name: Set up Python 3.12 | |
| run: uv python install 3.12 | |
| - name: Install dependencies | |
| run: uv sync --extra performance --group build | |
| - name: Build standard wheel | |
| run: uv build --wheel | |
| - name: Upload wheel artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: wheels-standard | |
| path: dist/*.whl | |
| build-wheels-mypyc: | |
| name: Build MyPyC wheels (${{ matrix.os }} py${{ matrix.python-version }}) | |
| runs-on: ${{ matrix.os }} | |
| needs: [detect-mode, wait-for-cheap-ci] | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: ${{ needs.detect-mode.outputs.matrix_mode == 'full' && fromJSON('["ubuntu-latest", "macos-latest", "windows-latest"]') || fromJSON('["ubuntu-latest"]') }} | |
| python-version: | |
| - ${{ github.event.inputs.python_version || '3.12' }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| - name: Export build constraints from uv.lock | |
| run: uv export --frozen --no-emit-project --no-hashes --format requirements.txt --output-file build-constraints.txt | |
| - name: Cache pip downloads | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/Library/Caches/pip | |
| ~\AppData\Local\pip\Cache | |
| key: pip-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('build-constraints.txt') }} | |
| restore-keys: | | |
| pip-${{ runner.os }}-py${{ matrix.python-version }}- | |
| - name: Cache mypyc build dir | |
| if: matrix.os == 'ubuntu-latest' | |
| uses: actions/cache@v5 | |
| with: | |
| path: /tmp/sqlspec-mypyc-build | |
| key: mypyc-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('sqlspec/**/*.py', 'pyproject.toml') }} | |
| restore-keys: | | |
| mypyc-${{ matrix.os }}-py${{ matrix.python-version }}- | |
| - name: Pre-warm Windows Defender exclusions | |
| if: matrix.os == 'windows-latest' | |
| shell: pwsh | |
| run: | | |
| # Defender's real-time scan on the freshly-extracted nuget-cpython | |
| # python.exe blocks virtualenv discovery past its 10s timeout. | |
| Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\pypa\cibuildwheel" -ErrorAction SilentlyContinue | |
| Add-MpPreference -ExclusionPath "$env:TEMP" -ErrorAction SilentlyContinue | |
| Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" -ErrorAction SilentlyContinue | |
| Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\wheelhouse" -ErrorAction SilentlyContinue | |
| Add-MpPreference -ExclusionProcess "python.exe" -ErrorAction SilentlyContinue | |
| - name: Install cibuildwheel | |
| run: python -m pip install cibuildwheel | |
| - name: Build wheels with cibuildwheel | |
| run: python -m cibuildwheel --output-dir wheelhouse | |
| env: | |
| CIBW_BUILD: "cp${{ matrix.python-version == '3.10' && '310' || matrix.python-version == '3.11' && '311' || matrix.python-version == '3.12' && '312' || matrix.python-version == '3.13' && '313' || matrix.python-version == '3.14' && '314' }}-*" | |
| CIBW_BUILD_VERBOSITY: 1 | |
| # PR validation builds host arch only — release covers full cross-arch matrix. | |
| CIBW_ARCHS_LINUX: "x86_64" | |
| CIBW_ARCHS_MACOS: "arm64" | |
| CIBW_ARCHS_WINDOWS: "AMD64" | |
| CIBW_SKIP: "cp39-win_arm64 *-musllinux*" | |
| CIBW_BEFORE_BUILD: "pip install -c {project}/build-constraints.txt hatch-mypyc hatchling" | |
| CIBW_TEST_REQUIRES: "cloud-sql-python-connector google-cloud-alloydb-connector" | |
| CIBW_TEST_COMMAND: >- | |
| python {project}/tools/scripts/mypyc_smoke.py --require-compiled | |
| # Linux PR validation exercises the same three-stage PGO path as publish.yml. | |
| CIBW_BEFORE_BUILD_LINUX: >- | |
| pip install -c {project}/build-constraints.txt hatch-mypyc hatchling && | |
| PGO_DIR=/tmp/sqlspec-pgo && | |
| BUILD_DIR=/tmp/sqlspec-mypyc-build && | |
| mkdir -p "$PGO_DIR" "$BUILD_DIR" && | |
| CFLAGS="-fprofile-generate=$PGO_DIR" | |
| HATCH_BUILD_HOOKS_ENABLE=1 | |
| HATCH_MYPYC_BUILD_DIR="$BUILD_DIR" | |
| MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 | |
| pip install {package} --no-build-isolation && | |
| pip install anyio pyarrow aiosqlite duckdb adbc-driver-sqlite adbc-driver-manager && | |
| python {project}/tools/scripts/pgo_training.py && | |
| pip uninstall -y sqlspec && | |
| rm -rf "$BUILD_DIR/build" "$BUILD_DIR/tmp" | |
| CIBW_ENVIRONMENT_LINUX: >- | |
| HATCH_BUILD_HOOKS_ENABLE=1 | |
| HATCH_MYPYC_BUILD_DIR=/tmp/sqlspec-mypyc-build | |
| MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 | |
| CFLAGS="-fprofile-use=/tmp/sqlspec-pgo -fprofile-correction -Wno-error=missing-profile -Wno-error=coverage-mismatch" | |
| # macOS + Windows: plain mypyc, no PGO. macOS PGO needs filter | |
| # deps absent from BEFORE_BUILD; MSVC PGO isn't validated. | |
| CIBW_ENVIRONMENT_MACOS: >- | |
| HATCH_BUILD_HOOKS_ENABLE=1 | |
| MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 | |
| CIBW_ENVIRONMENT_WINDOWS: >- | |
| HATCH_BUILD_HOOKS_ENABLE=1 | |
| MYPYC_OPT_LEVEL=3 MYPYC_DEBUG_LEVEL=0 MYPYC_MULTI_FILE=1 | |
| - name: Upload wheel artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: wheels-mypyc-${{ matrix.os }}-py${{ matrix.python-version }} | |
| path: wheelhouse/*.whl | |
| test-wheels: | |
| name: Test ${{ matrix.os }} py${{ matrix.python-version }} | |
| needs: [build-wheels-standard, build-wheels-mypyc, detect-mode] | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| os: ${{ needs.detect-mode.outputs.matrix_mode == 'full' && fromJSON('["ubuntu-latest", "windows-latest", "macos-latest"]') || fromJSON('["ubuntu-latest"]') }} | |
| python-version: | |
| - ${{ github.event.inputs.python_version || '3.12' }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| # PR builds have no release tag — derive expected version from | |
| # pyproject.toml so the version assertion below has something to check. | |
| - name: Resolve expected version | |
| shell: bash | |
| run: | | |
| v=$(grep -m1 '^version = ' pyproject.toml | sed -E 's/version = "(.*)"/\1/') | |
| echo "SQLSPEC_EXPECTED_VERSION=$v" >> "$GITHUB_ENV" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| - name: Set up Python ${{ matrix.python-version }} | |
| run: uv python install ${{ matrix.python-version }} | |
| - name: Download standard wheel artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: wheels-standard | |
| path: dist-standard/ | |
| - name: Download mypyc wheel artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: wheels-mypyc-* | |
| merge-multiple: true | |
| path: dist-mypyc/ | |
| - name: Test standard wheel installation | |
| shell: bash | |
| run: | | |
| uv venv test-standard --python ${{ matrix.python-version }} | |
| uv pip install --python test-standard --no-index --no-deps --find-links dist-standard/ "sqlspec==${SQLSPEC_EXPECTED_VERSION}" | |
| uv pip install --python test-standard "sqlspec==${SQLSPEC_EXPECTED_VERSION}" --find-links dist-standard/ | |
| uv run --no-project --python test-standard python -I -c "import sqlspec, os, sys; v=sqlspec.__version__; e=os.environ['SQLSPEC_EXPECTED_VERSION']; sys.exit(0 if v == e else (sys.stderr.write(f'version mismatch: got {v}, expected {e}\n') or 1))" | |
| uv run --no-project --python test-standard python -I -c "import sqlspec; print('Standard wheel OK')" | |
| - name: Test mypyc wheel installation | |
| shell: bash | |
| run: | | |
| uv venv test-mypyc --python ${{ matrix.python-version }} | |
| uv pip install --python test-mypyc --no-index --no-deps --find-links dist-mypyc/ "sqlspec==${SQLSPEC_EXPECTED_VERSION}" | |
| uv pip install --python test-mypyc "sqlspec==${SQLSPEC_EXPECTED_VERSION}" --find-links dist-mypyc/ | |
| uv run --no-project --python test-mypyc python -I -c "import sqlspec, os, sys; v=sqlspec.__version__; e=os.environ['SQLSPEC_EXPECTED_VERSION']; sys.exit(0 if v == e else (sys.stderr.write(f'version mismatch: got {v}, expected {e}\n') or 1))" | |
| uv run --no-project --python test-mypyc python -I tools/scripts/mypyc_smoke.py --require-compiled | |
| verify-packages: | |
| name: Verify package integrity | |
| needs: [build-source, build-wheels-standard, build-wheels-mypyc] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: "*" | |
| merge-multiple: true | |
| path: dist/ | |
| - name: List all built packages | |
| run: | | |
| echo "=== All built packages ===" | |
| find dist/ -name "*.whl" -o -name "*.tar.gz" | sort | |
| echo "=== Package count ===" | |
| find dist/ -name "*.whl" | wc -l | |
| find dist/ -name "*.tar.gz" | wc -l |