Fixes issue where Jupyter notebook cells couldn't be run (#14)

* Adds a script to fetch the requirements.txt for the correct version of JupyterHub being used

* Updates Dockerfile.spawn to properly install IPython and avoid issues running Jupyter notebooks

Author: ilumsden

2025-HPDC/docker/Dockerfile.spawn

@@ -24,11 +24,10 @@ COPY ./docker/requirements.txt /requirements.txt
 
 RUN . /opt/global_py_venv/bin/activate && \
     # Needed for some viz in thicket-tutorial
-    python3 -m pip install plotly[express] && \
-    python3 -m pip install -r /requirements.txt && \
-    # Covered in requirements.txt, so shouldn't be needed here
-    # python3 -m pip install ipython==7.34.0 ipykernel==6.25.1 && \
-    python3 -m IPython kernel install
+    /opt/global_py_venv/bin/python3 -m pip install plotly[express] && \
+    /opt/global_py_venv/bin/python3 -m pip install -r /requirements.txt && \
+    /opt/global_py_venv/bin/python3 -m pip install -U jupyter ipython ipykernel && \
+    /opt/global_py_venv/bin/python3 -m ipykernel install --prefix=/usr/local --name 'benchpark-tutorial-kernel'
 
 COPY ./docker/kitware-archive.sh /tmp
 
@@ -41,7 +40,12 @@ RUN /tmp/kitware-archive.sh && \
 COPY ./tutorial-code/thicket-tutorial/requirements.txt /tmp/thicket-tutorial/requirements.txt
 
 RUN . /opt/global_py_venv/bin/activate && \
-    python3 -m pip install -r /tmp/thicket-tutorial/requirements.txt
+    python3 -m pip install extrap scikit-learn seaborn
+# NOTE: we're not installing the dependencies for thicket-tutorial with
+#       requirements.txt because it hardcodes a version of IPython, which
+#       breaks all sorts of things. Instead, we extract the relevant dependencies
+#       and place them above.
+# python3 -m pip install -r /tmp/thicket-tutorial/requirements.txt
 
 COPY ./tutorial-code/caliper-tutorial/apps /tmp/caliper-tutorial/apps/
 COPY ./tutorial-code/caliper-tutorial/cmake /tmp/caliper-tutorial/cmake/
@@ -78,7 +82,16 @@ RUN cmake \
 # && \
 # rm -rf /tmp/build-xsbench
 
-RUN chmod -R 777 ~/ ${HOME}
+COPY ./tutorial-code/caliper-tutorial/tutorial ${HOME}/caliper-tutorial/
+COPY ./tutorial-code/thicket-tutorial/data/ ${HOME}/thicket-tutorial/data/
+COPY ./tutorial-code/thicket-tutorial/notebooks ${HOME}/thicket-tutorial/notebooks/
+COPY ./tutorial-code/thicket-tutorial/LICENSE ${HOME}/thicket-tutorial
+
+COPY tutorial-code/system-description/aws-tutorial ${HOME}/benchpark/systems/aws-tutorial
+COPY tutorial-code/system-description/AWS_Tutorial-c7i-EFA ${HOME}/benchpark/systems/all_hardware_descriptions/AWS_Tutorial-c7i-EFA
+
+RUN chown -R jovyan ${HOME}
+# RUN chmod -R 777 ~/ ${HOME}
 
 WORKDIR ${HOME}
 
@@ -91,14 +104,6 @@ USER ${NB_USER}
 ENV SHELL=/usr/bin/bash
 ENV FLUX_URI_RESOLVE_LOCAL=t
 
-COPY ./tutorial-code/caliper-tutorial/tutorial ${HOME}/caliper-tutorial/
-COPY ./tutorial-code/thicket-tutorial/data/ ${HOME}/thicket-tutorial/data/
-COPY ./tutorial-code/thicket-tutorial/notebooks ${HOME}/thicket-tutorial/notebooks/
-COPY ./tutorial-code/thicket-tutorial/LICENSE ${HOME}/thicket-tutorial
-
-COPY tutorial-code/system-description/aws-tutorial ${HOME}/benchpark/systems/aws-tutorial
-COPY tutorial-code/system-description/AWS_Tutorial-c7i-EFA ${HOME}/benchpark/systems/all_hardware_descriptions/AWS_Tutorial-c7i-EFA
-
 EXPOSE 8888
 ENTRYPOINT [ "tini", "--" ]
 

2025-HPDC/docker/get_requirements_for_jupyterhub_k8s.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+function usage {
+    echo "Usage: ./get_requirements_for_jupyterhub_k8s.sh <jupyterhub_k8s_version>"
+}
+
+if [[ $# -ne 1 ]]; then
+    usage
+    exit 1
+fi
+
+jh_k8s_version="$1"
+
+if [[ "$jh_k8s_version" == "--help" ]] || [[ "$jh_k8s_version" == "-h" ]]; then
+    usage
+    exit 0
+fi
+
+if ! command -v gh >/dev/null; then
+    echo "This script requires the GitHub CLI (i.e., the 'gh' command)!"
+    echo "Install and try again"
+    exit 1
+fi
+
+if ! command -v curl >/dev/null; then
+    echo "This script requires the 'curl' command!"
+    echo "Install and try again"
+    exit 1
+fi
+
+if ! command -v jq >/dev/null; then
+    echo "This script requires the 'jq' command!"
+    echo "Install and try again"
+    exit 1
+fi
+
+if ! command -v jq >/dev/null; then
+    echo "This script requires the 'wget' command!"
+    echo "Install and try again"
+    exit 1
+fi
+
+if ! command -v sed >/dev/null; then
+    echo "This script requires the 'sed' command!"
+    echo "Install and try again"
+    exit 1
+fi
+
+github_url="https://api.github.com/repos/jupyterhub/zero-to-jupyterhub-k8s/contents/images/hub/requirements.txt?ref=$jh_k8s_version"
+
+file_lookup_json=$(curl -JL \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $(gh auth token)" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    "$github_url")
+
+wget_url=$(echo "$file_lookup_json" | jq -r '.download_url')
+
+wget -O $(pwd)/requirements.txt $wget_url
+
+sed -i '' '/psycopg2/d' $(pwd)/requirements.txt