Merge pull request #2 from stackhpc/training/leafcloud-v2.3

Update for Slurm appliance v2.3

Author: priteau

.github/bin/get-s3-image.sh

@@ -17,7 +17,7 @@ if [ -n "$image_exists" ]; then
 else
     echo "Image $image_name not found in OpenStack. Getting it from S3."
 
-    wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga
+    wget https://leafcloud.store/swift/v1/AUTH_f39848421b2747148400ad8eeae8d536/$bucket_name/$image_name --progress=dot:giga
 
     echo "Uploading image $image_name to OpenStack..."
     openstack image create --file $image_name --disk-format qcow2 $image_name --progress

.github/workflows/extra.yml

@@ -8,6 +8,7 @@ on:
       - 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
       - 'ansible/roles/cuda/**'
+      - 'ansible/roles/slurm_recompile/**' # runs on cuda group
       - 'ansible/roles/lustre/**'
       - '.github/workflows/extra.yml'
   pull_request:
@@ -32,16 +33,17 @@ jobs:
           - image_name: openhpc-extra-RL8
             source_image_name_key: RL8 # key into environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
             inventory_groups: doca,cuda,lustre
-            volume_size: 30 # needed for cuda
+            volume_size: 35 # needed for cuda
           - image_name: openhpc-extra-RL9
             source_image_name_key: RL9
             inventory_groups: doca,cuda,lustre
-            volume_size: 30 # needed for cuda
+            volume_size: 35 # needed for cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
       - uses: actions/checkout@v2
@@ -94,7 +96,7 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          
+
           PACKER_LOG=1 packer build \
           -on-error=${{ vars.PACKER_ON_ERROR }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
@@ -103,7 +105,7 @@ jobs:
           -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
           -var "volume_size=${{ matrix.build.volume_size }}" \
           openstack.pkr.hcl
-        
+
       - name: Get created image names from manifest
         id: manifest
         run: |
@@ -121,7 +123,7 @@ jobs:
       - name: Make image usable for further builds
         run: |
           . venv/bin/activate
-          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}" || true
 
       - name: Delete image for automatically-run workflows
         run: |

.github/workflows/fatimage.yml

@@ -10,6 +10,11 @@ on:
             - LEAFCLOUD
             - SMS
             - ARCUS
+        cleanup_on_failure:
+          description: Cleanup Packer resources on failure
+          type: boolean
+          required: true
+          default: true
 
 jobs:
   openstack:
@@ -23,17 +28,18 @@ jobs:
       matrix: # build RL8, RL9
         build:
           - image_name: openhpc-RL8
-            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.qcow2
+            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.raw
             inventory_groups: control,compute,login,update
           - image_name: openhpc-RL9
-            source_image_name: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2
+            source_image_name: Rocky-9-GenericCloud-Base-9.6-20250531.0.x86_64.qcow2
             inventory_groups: control,compute,login,update
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
       LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
+      PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
       - uses: actions/checkout@v2
@@ -78,7 +84,7 @@ jobs:
           packer init .
 
           PACKER_LOG=1 packer build \
-          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -on-error=${{ github.event.inputs.cleanup_on_failure && 'cleanup' || 'abort' }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
           -var "source_image_name=${{ matrix.build.source_image_name }}" \
           -var "image_name=${{ matrix.build.image_name }}" \
@@ -102,7 +108,7 @@ jobs:
       - name: Make image usable for further builds
         run: |
           . venv/bin/activate
-          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}" || true
 
       - name: Upload manifest artifact
         uses: actions/upload-artifact@v4

.github/workflows/nightly-cleanup.yml

@@ -59,10 +59,34 @@ jobs:
       - name: Delete CI clusters
         run: |
           . venv/bin/activate
-          if [[ -z "${ci_clusters}" ]]; then
+          if [[ -z ${ci_clusters} ]]; then
             echo "No clusters to delete."
             exit 0
           fi
-          echo "Deleting clusters: ${ci_clusters}"
-          ./dev/delete-cluster.py ${ci_clusters} --force
+
+          for cluster_prefix in ${ci_clusters}
+          do
+            echo "Processing cluster: $cluster_prefix"
+
+            # Get all servers with the matching name for control node
+            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
+
+            # Get unique server names to avoid duplicate cleanup
+            UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq)
+            for name in $UNIQUE_NAMES; do
+              echo "Deleting cluster with control node: $name"
+
+              # Get the first matching server ID by name
+              server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1)
+
+              # Make sure server still exists (wasn't deleted earlier)
+              if ! openstack server show "$server" &>/dev/null; then
+                echo "Server $server no longer exists, skipping $name."
+                continue
+              fi
+
+              echo "Deleting cluster $cluster_prefix (server $server)..."
+              ./dev/delete-cluster.py $cluster_prefix --force
+            done
+          done
         shell: bash

.github/workflows/nightlybuild.yml

@@ -36,6 +36,7 @@ jobs:
       CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }}
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
       LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
+      PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/release-image.yml

@@ -0,0 +1,37 @@
+name: Release images
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - published # should work for both pre-releases and releases
+env:
+  IMAGE_PATH: environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
+jobs:
+  ci-image-release:
+    name: ci-image-release
+    runs-on: ubuntu-22.04
+    concurrency: ${{ github.workflow }}-${{ github.ref }}
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+          - RL8
+          - RL9
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Write s3cmd configuration
+        run: echo "${{ secrets.LEAFCLOUD_S3_CFG }}" > ~/.s3cfg
+
+      - name: Install s3cmd
+        run: |
+          sudo apt-get update
+          sudo apt-get --yes install s3cmd
+      
+      - name: Retrieve image name
+        run: |
+          TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
+          echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
+      
+      - name: Copy image from pre-release to release bucket
+        run: s3cmd cp s3://openhpc-images-prerelease/${{ env.TARGET_IMAGE }} s3://openhpc-images

.github/workflows/s3-image-sync.yml

@@ -1,4 +1,4 @@
-name: Upload CI-tested images to Arcus S3 and sync clouds
+name: Upload CI-tested images to Leafcloud S3 and sync clouds
 on:
   workflow_dispatch:
   push:
@@ -21,11 +21,12 @@ jobs:
 
       - name: Write s3cmd configuration
         run: |
-          echo "${{ secrets['ARCUS_S3_CFG'] }}" > ~/.s3cfg
+          echo "${{ secrets['LEAFCLOUD_S3_CFG'] }}" > ~/.s3cfg
         shell: bash
 
       - name: Install s3cmd
         run: |
+          sudo apt-get update
           sudo apt-get --yes install s3cmd
       
       - name: Cleanup S3 bucket
@@ -72,23 +73,41 @@ jobs:
 
       - name: Write s3cmd configuration
         run: |
-          echo "${{ secrets['ARCUS_S3_CFG'] }}" > ~/.s3cfg
+          echo "${{ secrets['LEAFCLOUD_S3_CFG'] }}" > ~/.s3cfg
         shell: bash
 
-      - name: Install s3cmd
+      - name: Install s3cmd and qemu-utils
         run: |
-          sudo apt-get --yes install s3cmd
+          sudo apt-get update
+          sudo apt-get --yes install s3cmd qemu-utils
 
       - name: Retrieve image name
         run: |
           TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
           echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
         shell: bash
 
+      - name: Clear up some space on runner
+        run: |
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo apt-get clean
+          df -h
+
       - name: Download image to runner
         run: |
           . venv/bin/activate
-          openstack image save --file ${{ env.TARGET_IMAGE }} ${{ env.TARGET_IMAGE }}
+          openstack image save --file "${{ env.TARGET_IMAGE }}.raw" "${{ env.TARGET_IMAGE }}"
+          df -h
+        shell: bash
+
+      - name: Convert image to QCOW2
+        run: |
+          . venv/bin/activate
+          qemu-img convert -f raw -O qcow2 -c "${{ env.TARGET_IMAGE }}.raw" "${{ env.TARGET_IMAGE }}"
         shell: bash
 
       - name: Upload Image to S3

.github/workflows/stackhpc.yml

@@ -9,8 +9,7 @@ on:
       - '**'
       - '!dev/**'
       - 'dev/setup-env.sh'
-      - '!docs/**'
-      - '!README.md'
+      - '!**.md'
       - '!.gitignore'
       - '!.github/workflows/'
       - '.github/workflows/stackhpc'
@@ -19,8 +18,7 @@ on:
       - '**'
       - '!dev/**'
       - 'dev/setup-env.sh'
-      - '!docs/**'
-      - '!README.md'
+      - '!**.md'
       - '!.gitignore'
       - '!.github/workflows/'
       - '.github/workflows/stackhpc'
@@ -91,9 +89,9 @@ jobs:
         run: dev/setup-env.sh
 
       - name: Install OpenTofu
-        uses: opentofu/setup-opentofu@v1
+        uses: opentofu/setup-opentofu@v1.0.5
         with:
-          tofu_version: 1.6.2
+          tofu_version: 1.9.0
 
       - name: Initialise tofu
         run: tofu init
@@ -230,6 +228,16 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
+      - name: Delete possible volume snapshot from slurm upgrade
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate  
+          if [ -n "$SNAPSHOT" ]
+          then
+              echo Deleting $SNAPSHOT
+              openstack volume snapshot delete $SNAPSHOT
+          fi
+
       - name: Delete infrastructure
         run: |
           . venv/bin/activate

.github/workflows/upload-release-image.yml.sample

@@ -19,7 +19,7 @@ on:
     inputs:
       image_name:
         type: string
-        description: Image name from: (https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/{BUCKET_NAME})
+        description: Image name from: (https://leafcloud.store/swift/v1/AUTH_f39848421b2747148400ad8eeae8d536/{BUCKET_NAME})
         required: true
       bucket_name:
         type: choice

.gitignore

@@ -5,3 +5,4 @@ venv
 *.pyc
 packer/openhpc2
 .vscode
+requirements.yml.last