Drop Python 3.8; upgrade to Pandas 2 (#813)

Author: crusaderky

.github/workflows/software-environments-nightly.yml

@@ -35,16 +35,15 @@ jobs:
         pytest_args: [tests]
         runtime-version: [upstream, latest]
         include:
-          # Run stability tests on Python 3.8
+          # Run stability tests on the lowest and highest versions of Python only
           - pytest_args: tests/stability
-            python-version: "3.8"
+            python-version: "3.9"
             runtime-version: upstream
             os: ubuntu-latest
           - pytest_args: tests/stability
-            python-version: "3.8"
+            python-version: "3.9"
             runtime-version: latest
             os: ubuntu-latest
-          # Run stability tests on Python 3.10
           - pytest_args: tests/stability
             python-version: "3.10"
             runtime-version: upstream

.github/workflows/software-environments-stable.yml

@@ -9,11 +9,19 @@ repos:
       hooks:
       - id: isort
         language_version: python3
+  -   repo: https://github.com/asottile/pyupgrade
+      rev: v3.3.2
+      hooks:
+      - id: pyupgrade
+        args:
+          - --py39-plus
   -   repo: https://github.com/psf/black
       rev: 23.1.0
       hooks:
       - id: black
         language_version: python3
+        args:
+          - --target-version=py39
         exclude: ^alembic/versions/
   -   repo: https://github.com/pycqa/flake8
       rev: 6.0.0

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ dependencies:
     - pip
     - coiled >=0.2.54
     - numpy ==1.23.5
-    - pandas ==1.5.3  # 2.0.0 requires a newer version of xarray
+    - pandas ==2.0.1
     - dask ==2023.4.0
     - distributed ==2023.4.0
     - fsspec ==2023.3.0
@@ -25,7 +25,7 @@ dependencies:
     - scikit-learn ==1.2.2
     - ipycytoscape ==1.3.3
     - click ==8.1.3
-    - xarray ==2023.1.0  # Last version supporting Python 3.8
+    - xarray ==2023.4.2
     - zarr ==2.14.2
     - cftime ==1.6.2
     - msgpack-python ==1.0.5

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ dependencies:
     - pip
     - coiled >=0.2.54
     - numpy ==1.23.5
-    - pandas ==1.5.3  # 2.0.0 requires a newer version of xarray
+    - pandas ==2.0.1
     - dask ==2023.4.0
     - distributed ==2023.4.0
     - fsspec ==2023.3.0
@@ -30,7 +30,7 @@ dependencies:
     - scikit-learn ==1.2.2
     - ipycytoscape ==1.3.3
     - click ==8.1.3
-    - xarray ==2023.1.0  # Last version supporting Python 3.8
+    - xarray ==2023.4.2
     - zarr ==2.14.2
     - cftime ==1.6.2
     - msgpack-python ==1.0.5

AB_environments/AB_baseline.conda.yaml

@@ -34,16 +34,16 @@ channels:
 dependencies:
     - python =3.9
     - <copy-paste from recipe/meta.yaml, minus bits you want to change>
-    - pip:
-      - dask ==2023.4.1
-      - distributed ==2023.4.1
+    # Changes from the recipe start here
+    - dask ==2023.4.1
+    - distributed ==2023.4.1
 ```
 Instead of published packages, you could also use arbitrary git hashes of arbitrary
 forks, e.g.
 
 ```yaml
     - pip:
-      - dask ==2023.4.2
+      - git+https://github.com/dask/dask@b85bf5be72b02342222c8a0452596539fce19bce
       - git+https://github.com/yourname/distributed@803c624fcef99e3b6f3f1c5bce61a2fb4c9a1717
 ```
 
@@ -143,48 +143,19 @@ etc.
 
 
 ### Complete example
-You want to test the impact of disabling work stealing. You'll create at least 4 files:
+You want to test the impact of disabling work stealing on the latest version of dask.
+You'll create at least 4 files:
 
 - `AB_environments/AB_baseline.conda.yaml`:
 ```yaml
 channels:
   - conda-forge
 dependencies:
     - python =3.9
-    - pip
-    - coiled >=0.2.54
-    - numpy ==1.23.5
-    - pandas ==1.5.3
-    - dask ==2023.4.0
-    - distributed ==2023.4.0
-    - fsspec ==2023.3.0
-    - s3fs ==2023.3.0
-    - gcsfs ==2023.3.0
-    - pyarrow ==11.0.0
-    - jupyterlab ==3.6.2
-    - dask-labextension ==6.1.0
-    - lz4 ==4.3.2
-    - ipywidgets ==8.0.4
-    - numba ==0.56.4
-    - scikit-learn ==1.2.2
-    - ipycytoscape ==1.3.3
-    - click ==8.1.3
-    - xarray ==2023.1.0
-    - zarr ==2.14.2
-    - cftime ==1.6.2
-    - msgpack-python ==1.0.5
-    - cloudpickle ==2.2.1
-    - tornado ==6.2
-    - toolz ==0.12.0
-    - zict ==3.0.0
-    - xgboost ==1.7.4
-    - dask-ml ==2023.3.24
-    - openssl >1.1.0g
-    - optuna ==3.1.0
-    - scipy ==1.10.1
-    - sqlalchemy ==1.4.46
-    - pynvml ==11.5.0
-    - bokeh ==3.1.0
+    - coiled
+    - dask
+    - distributed
+    - s3fs
 ```
 - `AB_environments/AB_baseline.dask.yaml`: (empty file)
 - `AB_environments/AB_baseline.cluster.yaml`: (empty file)
@@ -201,9 +172,7 @@ distributed:
 repeat: 5
 test_null_hypothesis: true
 targets:
-  - tests/runtime
   - tests/benchmarks
-  - tests/stability
 h2o_datasets:
   - 5 GB (parquet+pyarrow)
 max_parallel:
@@ -213,8 +182,9 @@ max_parallel:
 
 ### 6. Run CI
 - `git push`. Note: you should *not* open a Pull Request. 
-- Open https://github.com/coiled/benchmarks/actions/workflows/ab_tests.yml and wait
-  for the run to complete.
+- Open [the GitHub Actions tab] 
+  (https://github.com/coiled/benchmarks/actions/workflows/ab_tests.yml)
+  and wait for the run to complete.
 - Open the run from the link above. In the Summary tab, scroll down and download the
   `static-dashboard` artifact. 
   Note: artifacts will appear only after the run is complete.

AB_environments/AB_sample.conda.yaml

@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 import sys
-from typing import Iterator
+from collections.abc import Iterator
 
 import yaml
 

AB_environments/README.md

@@ -9,7 +9,7 @@
 import pathlib
 from collections.abc import Callable
 from textwrap import dedent
-from typing import Any, Literal, NamedTuple, Optional
+from typing import Any, Literal, NamedTuple
 from urllib.parse import quote
 
 import altair
@@ -135,7 +135,7 @@ def confidence(
     pivot_groups = pivot.groupby(["fullname", "fullname_no_category"])[["diff"]]
     x_neg = numpy.linspace(-0.8, 0, 17)
     x_pos = numpy.linspace(0, 0.8, 17)
-    conf_neg, conf_pos = [
+    conf_neg, conf_pos = (
         # DataFrame with 1 row per element of x_neg/x_pos and columns
         # [fullname, fullname_no_category, x, xlabel, p, color]
         (
@@ -147,7 +147,7 @@ def confidence(
             (x_neg, "<", operator.lt, -1),
             (x_pos, ">", operator.gt, 1),
         )
-    ]
+    )
     return pandas.concat([conf_neg, conf_pos], axis=0)
 
 
@@ -614,7 +614,7 @@ def make_details_html_report(
     )
 
 
-def make_grafana_url(cluster_name, start, end) -> Optional[str]:
+def make_grafana_url(cluster_name, start, end) -> str | None:
     if cluster_name:
         # Add some padding to compensate for clock differences between
         # GitHub actions and Prometheus, as well for sample granularity
@@ -667,7 +667,7 @@ def make_index_html_report(
 
 def runtime_sort_key(runtime: str) -> tuple:
     """Runtimes are in the format coiled-<coiled-runtime version>-py<python version>
-    e.g. coiled-latest-py3.8
+    e.g. coiled-latest-py3.9
 
     Sort them by coiled-runtime and python version, both descending.
     """