From 6e6c4bcb7630af4f48abf09120ce062745ac3e98 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 31 Oct 2024 10:44:04 +0100 Subject: [PATCH 1/3] Pin and automate doing isolated bumps of hub image dependencies' major versions --- .github/dependabot.yaml | 12 +++++++ .github/workflows/watch-dependencies.yaml | 16 ++++----- ci/refreeze | 2 +- images/hub/README.md | 2 +- images/hub/requirements.in | 32 ------------------ images/hub/unfrozen/requirements.txt | 33 +++++++++++++++++++ images/singleuser-sample/requirements.in | 17 ---------- .../unfrozen/requirements.txt | 17 ++++++++++ 8 files changed, 72 insertions(+), 59 deletions(-) delete mode 100644 images/hub/requirements.in create mode 100644 images/hub/unfrozen/requirements.txt delete mode 100644 images/singleuser-sample/requirements.in create mode 100644 images/singleuser-sample/unfrozen/requirements.txt diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 40e0649746..b2deaded0b 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -15,3 +15,15 @@ updates: interval: monthly time: "05:00" timezone: Etc/UTC + - package-ecosystem: pip + directory: /images/hub/unfrozen + labels: [breaking] + groups: + major-versions: + update-types: [major] + exclude-patterns: + - jupyterhub # bumped by other automation + schedule: + interval: daily + time: "05:00" + timezone: Etc/UTC diff --git a/.github/workflows/watch-dependencies.yaml b/.github/workflows/watch-dependencies.yaml index b65e70fede..5f1907b9c5 100644 --- a/.github/workflows/watch-dependencies.yaml +++ b/.github/workflows/watch-dependencies.yaml @@ -4,7 +4,7 @@ # - Watch multiple images tags referenced in values.yaml to match the latest # image tag. # -# - Watch the jupyterhub pinning in images/*/requirements.in to match the +# - Watch the jupyterhub pinning in images/*/unfrozen/requirements.txt to match the # latest jupyterhub version available on PyPI, and if doing this, also # refreeze images/*/requirements.txt. # @@ -20,7 +20,7 @@ name: Watch dependencies on: push: paths: - - "images/*/requirements.in" + - "images/*/unfrozen/requirements.txt" - ".github/workflows/watch-dependencies.yaml" branches: ["main"] schedule: @@ -142,10 +142,10 @@ jobs: - name: Install Python dependencies run: pip install packaging requests - - name: Get images/hub/requirements.in pinned version of jupyterhub + - name: Get images/hub/unfrozen/requirements.txt pinned version of jupyterhub id: local run: | - local_version=$(cat images/hub/requirements.in | grep 'jupyterhub==' | sed 's/jupyterhub==//') + local_version=$(cat images/hub/unfrozen/requirements.txt | grep 'jupyterhub==' | sed 's/jupyterhub==//') echo "version=$local_version" >> $GITHUB_OUTPUT - name: Get latest version of jupyterhub @@ -168,11 +168,11 @@ jobs: if: steps.local.outputs.version != steps.latest.outputs.version run: | for img in hub singleuser-sample; do - sed --in-place 's/jupyterhub==${{ steps.local.outputs.version }}/jupyterhub==${{ steps.latest.outputs.version }}/g' images/$img/requirements.in + sed --in-place 's/jupyterhub==${{ steps.local.outputs.version }}/jupyterhub==${{ steps.latest.outputs.version }}/g' images/$img/unfrozen/requirements.txt done sed --in-place 's/appVersion: "${{ steps.local.outputs.version }}"/appVersion: "${{ steps.latest.outputs.version }}"/g' jupyterhub/Chart.yaml - - name: Refreeze images/*/requirements.txt based on images/*/requirements.in + - name: Refreeze images/*/requirements.txt based on images/*/unfrozen/requirements.txt if: steps.local.outputs.version != steps.latest.outputs.version run: ci/refreeze @@ -208,7 +208,7 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Refreeze images/*/requirements.txt based on images/*/requirements.in + - name: Refreeze images/*/requirements.txt based on images/*/unfrozen/requirements.txt run: ci/refreeze - name: git diff @@ -227,4 +227,4 @@ jobs: title: "hub image: refreeze requirements.txt" body: >- The hub image's requirements.txt has been refrozen based on - requirements.in. + unfrozen/requirements.txt. diff --git a/ci/refreeze b/ci/refreeze index a1244b0814..637ac2eea8 100755 --- a/ci/refreeze +++ b/ci/refreeze @@ -14,6 +14,6 @@ for img in ${IMAGES}; do --workdir=/io \ --user=root \ python:3.12-bookworm \ - sh -c 'pip install pip-tools==7.* && pip-compile --allow-unsafe --strip-extras --upgrade' + sh -c 'pip install pip-tools==7.* && pip-compile --allow-unsafe --strip-extras --upgrade --output-file=requirements.txt unfrozen/requirements.txt' popd done diff --git a/images/hub/README.md b/images/hub/README.md index 4dd81fae13..8aed95b86b 100644 --- a/images/hub/README.md +++ b/images/hub/README.md @@ -3,7 +3,7 @@ The Dockerfile in this folder is built by [chartpress](https://github.com/jupyterhub/chartpress#readme), using the requirements.txt file. The requirements.txt file is updated based on the -requirements.in file using [`pip-compile`](https://pip-tools.readthedocs.io). +unfrozen/requirements.txt file using [`pip-compile`](https://pip-tools.readthedocs.io). ## How to update requirements.txt diff --git a/images/hub/requirements.in b/images/hub/requirements.in deleted file mode 100644 index fae1f74f69..0000000000 --- a/images/hub/requirements.in +++ /dev/null @@ -1,32 +0,0 @@ -# This file is the input to requirements.txt, -# which is a frozen version of this. To update -# requirements.txt, use the "Run workflow" button at -# https://github.com/jupyterhub/zero-to-jupyterhub-k8s/actions/workflows/watch-dependencies.yaml -# that will also update the jupyterhub version if needed. -# README.md file. - -# JupyterHub itself, update this version pinning by running the workflow -# mentioned above. -jupyterhub==5.2.1 - -## Authenticators -jupyterhub-firstuseauthenticator>=1 -jupyterhub-hmacauthenticator -jupyterhub-ldapauthenticator>=2.0.1 -jupyterhub-ltiauthenticator!=1.3.0 -jupyterhub-nativeauthenticator -jupyterhub-tmpauthenticator -oauthenticator[googlegroups,mediawiki] - -## Kubernetes spawner -jupyterhub-kubespawner>=7.0.0 - -## Other optional dependencies for additional features -pymysql # mysql -psycopg2 # postgres -pycurl # internal http requests handle more load with pycurl -sqlalchemy-cockroachdb # cocroachdb -statsd # statsd metrics collection (TODO: remove soon, since folks use prometheus) - -# The idle culler service -jupyterhub-idle-culler diff --git a/images/hub/unfrozen/requirements.txt b/images/hub/unfrozen/requirements.txt new file mode 100644 index 0000000000..3d159e3b6e --- /dev/null +++ b/images/hub/unfrozen/requirements.txt @@ -0,0 +1,33 @@ +# This file is the input to requirements.txt, which is a frozen version of this. +# +# To update: +# - the jupyterhub version or the frozen requirements.txt file, use the +# "Run workflow" button at https://github.com/jupyterhub/zero-to-jupyterhub-k8s/actions/workflows/watch-dependencies.yaml. +# - the major version pins in this file, trigger dependabot to run via +# https://github.com/jupyterhub/zero-to-jupyterhub-k8s/network/updates +# + +# JupyterHub itself +jupyterhub==5.2.1 + +# JupyterHub Spawner, kubernetes specific +jupyterhub-kubespawner==7.* + +# JupyterHub Authenticator choices +jupyterhub-firstuseauthenticator==1.* +jupyterhub-hmacauthenticator==1.* +jupyterhub-ldapauthenticator==2.* +jupyterhub-ltiauthenticator==1.* +jupyterhub-nativeauthenticator==1.* +jupyterhub-tmpauthenticator==1.* +oauthenticator[googlegroups,mediawiki]==17.* + +# JupyterHub service shutting servers after a period of inactivity +jupyterhub-idle-culler==1.* + +# Other optional dependencies for additional features +pymysql==1.* # mysql +psycopg2==2.* # postgres +pycurl==7.* # internal http requests handle more load with pycurl +sqlalchemy-cockroachdb==2.* # cocroachdb +statsd==4.* # statsd metrics collection (TODO: remove soon, since folks use prometheus) diff --git a/images/singleuser-sample/requirements.in b/images/singleuser-sample/requirements.in deleted file mode 100644 index 01af3e693f..0000000000 --- a/images/singleuser-sample/requirements.in +++ /dev/null @@ -1,17 +0,0 @@ -# This file is the input to requirements.txt, -# which is a frozen version of this. To update -# requirements.txt, use the "Run workflow" button at -# https://github.com/jupyterhub/zero-to-jupyterhub-k8s/actions/workflows/watch-dependencies.yaml -# that will also update the jupyterhub version if needed. -# README.md file. - -# JupyterHub itself, update this version pinning by running the workflow -# mentioned above. -jupyterhub==5.2.1 - -# UI -jupyterlab -nbclassic - -# plugins -nbgitpuller diff --git a/images/singleuser-sample/unfrozen/requirements.txt b/images/singleuser-sample/unfrozen/requirements.txt new file mode 100644 index 0000000000..cf60156871 --- /dev/null +++ b/images/singleuser-sample/unfrozen/requirements.txt @@ -0,0 +1,17 @@ +# This file is the input to requirements.txt, which is a frozen version of this. +# +# To update: +# - the jupyterhub version or the frozen requirements.txt file, use the +# "Run workflow" button at https://github.com/jupyterhub/zero-to-jupyterhub-k8s/actions/workflows/watch-dependencies.yaml. +# + +# JupyterHub itself, update this version pinning by running the workflow +# mentioned above. +jupyterhub==5.2.1 + +# UI +jupyterlab +nbclassic + +# plugins +nbgitpuller From 0d46c7752d3d8b306f4ae0cbf4e680c29bdf3f72 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 31 Oct 2024 11:51:30 +0100 Subject: [PATCH 2/3] Run ci/refreeze --- images/hub/requirements.txt | 30 +++++++++++------------ images/singleuser-sample/requirements.txt | 8 +++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/images/hub/requirements.txt b/images/hub/requirements.txt index d14a2e916d..55c1c95171 100644 --- a/images/hub/requirements.txt +++ b/images/hub/requirements.txt @@ -81,7 +81,7 @@ jupyter-events==0.10.0 # via jupyterhub jupyterhub==5.2.1 # via - # -r requirements.in + # -r unfrozen/requirements.txt # jupyterhub-firstuseauthenticator # jupyterhub-kubespawner # jupyterhub-ldapauthenticator @@ -89,21 +89,21 @@ jupyterhub==5.2.1 # jupyterhub-nativeauthenticator # oauthenticator jupyterhub-firstuseauthenticator==1.1.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-hmacauthenticator==1.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-idle-culler==1.4.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-kubespawner==7.0.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-ldapauthenticator==2.0.1 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-ltiauthenticator==1.6.2 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-nativeauthenticator==1.3.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterhub-tmpauthenticator==1.0.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt kubernetes-asyncio==31.1.0 # via jupyterhub-kubespawner ldap3==2.9.1 @@ -121,7 +121,7 @@ multidict==6.1.0 mwoauth==0.4.0 # via oauthenticator oauthenticator==17.1.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt oauthlib==3.2.2 # via # jupyterhub @@ -141,7 +141,7 @@ prometheus-client==0.21.0 propcache==0.2.0 # via yarl psycopg2==2.9.10 - # via -r requirements.in + # via -r unfrozen/requirements.txt pyasn1==0.6.1 # via # ldap3 @@ -152,7 +152,7 @@ pyasn1-modules==0.4.1 pycparser==2.22 # via cffi pycurl==7.45.3 - # via -r requirements.in + # via -r unfrozen/requirements.txt pydantic==2.9.2 # via jupyterhub pydantic-core==2.23.4 @@ -163,7 +163,7 @@ pyjwt==2.9.0 # mwoauth # oauthenticator pymysql==1.1.1 - # via -r requirements.in + # via -r unfrozen/requirements.txt python-dateutil==2.9.0.post0 # via # arrow @@ -224,9 +224,9 @@ sqlalchemy==2.0.36 # jupyterhub # sqlalchemy-cockroachdb sqlalchemy-cockroachdb==2.0.2 - # via -r requirements.in + # via -r unfrozen/requirements.txt statsd==4.0.1 - # via -r requirements.in + # via -r unfrozen/requirements.txt text-unidecode==1.3 # via python-slugify tornado==6.4.1 diff --git a/images/singleuser-sample/requirements.txt b/images/singleuser-sample/requirements.txt index 4e5f6afa52..54decdb723 100644 --- a/images/singleuser-sample/requirements.txt +++ b/images/singleuser-sample/requirements.txt @@ -136,9 +136,9 @@ jupyter-server==2.14.2 jupyter-server-terminals==0.5.3 # via jupyter-server jupyterhub==5.2.1 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterlab==4.3.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt jupyterlab-pygments==0.3.0 # via nbconvert jupyterlab-server==2.27.3 @@ -157,7 +157,7 @@ matplotlib-inline==0.1.7 mistune==3.0.2 # via nbconvert nbclassic==1.1.0 - # via -r requirements.in + # via -r unfrozen/requirements.txt nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -168,7 +168,7 @@ nbformat==5.10.4 # nbclient # nbconvert nbgitpuller==1.2.1 - # via -r requirements.in + # via -r unfrozen/requirements.txt nest-asyncio==1.6.0 # via # ipykernel From 70e33c10d625e733313f2306344462a85305a1b8 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 1 Nov 2024 09:04:09 +0100 Subject: [PATCH 3/3] remove major pins for transitive hub dependencies avoids PRs labeling them as 'breaking' Co-authored-by: Erik Sundell --- images/hub/unfrozen/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/images/hub/unfrozen/requirements.txt b/images/hub/unfrozen/requirements.txt index 3d159e3b6e..241c7990df 100644 --- a/images/hub/unfrozen/requirements.txt +++ b/images/hub/unfrozen/requirements.txt @@ -26,8 +26,8 @@ oauthenticator[googlegroups,mediawiki]==17.* jupyterhub-idle-culler==1.* # Other optional dependencies for additional features -pymysql==1.* # mysql -psycopg2==2.* # postgres -pycurl==7.* # internal http requests handle more load with pycurl -sqlalchemy-cockroachdb==2.* # cocroachdb -statsd==4.* # statsd metrics collection (TODO: remove soon, since folks use prometheus) +pymysql # mysql +psycopg2 # postgres +pycurl # internal http requests handle more load with pycurl +sqlalchemy-cockroachdb # cocroachdb +statsd # statsd metrics collection (TODO: remove soon, since folks use prometheus)