Use runsOn's magic cache to speed up CI tests

.github/workflows/create-lint-wf.yml

@@ -22,10 +22,79 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  prepare-cache:
+    name: Pre-build caches for lint jobs
+    runs-on:
+      - runs-on=${{ github.run_id }}-prepare-cache
+      - runner=2cpu-linux-x64
+      - extras=s3-cache
+    strategy:
+      matrix:
+        NXF_VER:
+          - "25.04.0"
+          - "latest-everything"
+    steps:
+      - name: Enable Magic Cache
+        uses: runs-on/action@v2
+
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        name: Check out source-code repository
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
+        with:
+          python-version: "3.14"
+          cache: pip
+
+      - name: Cache Python dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Cache Nextflow assets
+        uses: actions/cache@v4
+        with:
+          path: ~/.nextflow
+          key: ${{ runner.os }}-nextflow-${{ matrix.NXF_VER }}-${{ matrix.NXF_VER == 'latest-everything' && steps.date.outputs.date || 'stable' }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-${{ matrix.NXF_VER }}-
+
+      - name: Cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-precommit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-precommit-
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: ${{ matrix.NXF_VER }}
+
+      - name: Install pre-commit hooks
+        run: |
+          pip install pre-commit
+          pre-commit install-hooks
+
   MakeTestWorkflow:
     runs-on:
       - runs-on=${{ github.run_id }}-make-test-worfklow
       - runner=4cpu-linux-x64
+      - extras=s3-cache
+    needs: prepare-cache
     env:
       NXF_ANSI_LOG: false
 
@@ -35,6 +104,9 @@ jobs:
           - "25.04.0"
           - "latest-everything"
     steps:
+      - name: Enable Magic Cache
+        uses: runs-on/action@v2
+
       - name: go to subdirectory and change nextflow workdir
         run: |
           mkdir -p create-lint-wf
@@ -52,11 +124,39 @@ jobs:
           python-version: "3.14"
           cache: pip
 
+      - name: Cache Python dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .
 
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Cache Nextflow assets
+        uses: actions/cache@v4
+        with:
+          path: ~/.nextflow
+          key: ${{ runner.os }}-nextflow-${{ matrix.NXF_VER }}-${{ matrix.NXF_VER == 'latest-everything' && steps.date.outputs.date || 'stable' }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-${{ matrix.NXF_VER }}-
+
+      - name: Cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-precommit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-precommit-
+
       - name: run create-lint-wf
         uses: ./.github/actions/create-lint-wf
         with:

.github/workflows/create-test-lint-wf-template.yml

@@ -39,11 +39,90 @@ jobs:
           echo "matrix=$(yq '.[].features | keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config")' nf_core/pipelines/create/template_features.yml | \
           yq 'flatten | tojson(0)' -)" >> $GITHUB_OUTPUT
 
+  prepare-cache:
+    name: Pre-build caches for matrix jobs
+    runs-on:
+      - runs-on=${{ github.run_id }}-prepare-cache
+      - runner=2cpu-linux-x64
+      - extras=s3-cache
+    steps:
+      - name: Enable Magic Cache
+        uses: runs-on/action@v2
+
+      - name: Check out source-code repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
+        with:
+          python-version: "3.14"
+          cache: "pip"
+
+      - name: Cache Python dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Cache Nextflow assets
+        uses: actions/cache@v4
+        with:
+          path: ~/.nextflow
+          key: ${{ runner.os }}-nextflow-latest-everything-${{ steps.date.outputs.date }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-latest-everything-
+
+      - name: Cache nf-test
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.nf-test
+            ~/.pdiff
+          key: ${{ runner.os }}-nftest-0.9.3
+          restore-keys: |
+            ${{ runner.os }}-nftest-
+
+      - name: Cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-precommit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-precommit-
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: latest-everything
+
+      - name: Install nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: "0.9.3"
+          install-pdiff: true
+
+      - name: Install pre-commit hooks
+        run: |
+          pip install pre-commit
+          pre-commit install-hooks
+
   RunTestWorkflow:
     runs-on:
       - runs-on=${{ github.run_id }}-run-test-worfklow
       - runner=4cpu-linux-x64
-    needs: prepare-matrix
+      - extras=s3-cache
+    needs: [prepare-matrix, prepare-cache]
     env:
       NXF_ANSI_LOG: false
 
@@ -55,19 +134,53 @@ jobs:
       fail-fast: false
 
     steps:
+      - name: Enable Magic Cache
+        uses: runs-on/action@v2
+
       - name: Check out source-code repository
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
 
       - name: Set up Python 3.14
         uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
         with:
           python-version: "3.14"
+          cache: "pip"
+
+      - name: Cache Python dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
 
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
           pip install .
 
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Cache Nextflow assets
+        uses: actions/cache@v4
+        with:
+          path: ~/.nextflow
+          key: ${{ runner.os }}-nextflow-latest-everything-${{ steps.date.outputs.date }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-latest-everything-
+
+      - name: Cache nf-test
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.nf-test
+            ~/.pdiff
+          key: ${{ runner.os }}-nftest-0.9.3
+          restore-keys: |
+            ${{ runner.os }}-nftest-
+
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
@@ -79,6 +192,14 @@ jobs:
           version: "0.9.3"
           install-pdiff: true
 
+      - name: Cache pre-commit
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ runner.os }}-precommit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-precommit-
+
       # Create template files
       - name: Create template skip ${{ matrix.TEMPLATE }}
         run: |