From 3a631ec0007429dba0e6a20d6764b20d09f3912e Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Fri, 2 May 2025 15:02:03 +0100 Subject: [PATCH 1/4] Add ubuntu upgrade process on CI-AIO --- .github/workflows/stackhpc-all-in-one.yml | 23 ++++++++++++ .github/workflows/stackhpc-pull-request.yml | 39 ++++++++++----------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index e5b0a0ca5..00592c878 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -326,6 +326,29 @@ jobs: env: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Upgrade host OS + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + -e KAYOBE_PATH=/stack/kayobe-automation-env/src/kayobe \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/script-run.sh tools/ubuntu-upgrade-overcloud.sh all + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade && inputs.os_release == 'jammy' + + - name: Deploy services that are built for later OS + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade && inputs.os_release == 'jammy' + - name: Run upgrade prerequisites run: | docker run -t --rm \ diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index 264246551..5d71d0f74 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -181,26 +181,25 @@ jobs: secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} - # Test two upgrade scenarios: Ubuntu Noble OVS and Rocky 9 OVN. - - # On hold until Noble support lands in stackhpc/2024.1 - # all-in-one-upgrade-ubuntu-noble-ovs: - # name: aio upgrade (Ubuntu Noble OVS) - # needs: - # - check-changes - # - build-kayobe-image - # uses: ./.github/workflows/stackhpc-all-in-one.yml - # with: - # kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} - # os_distribution: ubuntu - # os_release: noble - # ssh_username: ubuntu - # neutron_plugin: ovs - # OS_CLOUD: openstack - # if: ${{ needs.check-changes.outputs.aio == 'true' }} - # upgrade: true - # secrets: inherit - # if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + # Test two upgrade scenarios: Ubuntu Jammy to Noble OVN and Rocky 9 OVN. + + all-in-one-upgrade-ubuntu-jammy-to-noble-ovn: + name: aio upgrade (Ubuntu Jammy to Noble OVN) + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: ubuntu + os_release: jammy + ssh_username: ubuntu + neutron_plugin: ovn + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + upgrade: true + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} all-in-one-upgrade-rocky-9-ovn: name: aio upgrade (Rocky 9 OVN) From 40010dcd6339df945066cb8f6962490de61dc838 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Tue, 20 May 2025 09:19:13 +0100 Subject: [PATCH 2/4] Add docker prune after upgrade --- .github/workflows/stackhpc-all-in-one.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 00592c878..7674ba849 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -414,6 +414,11 @@ jobs: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} if: inputs.upgrade + - name: Prune unused docker images + run: | + docker image prune -af + if: inputs.upgrade + - name: Tempest tests id: tempest run: | From 3be434de431efc88271a8e4e3a863a9fd4f6a182 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Thu, 29 May 2025 15:55:32 +0100 Subject: [PATCH 3/4] Bump lv_audit LVM size to 500MB for CI-AIO Rebooting from Ubuntu upgrade creates audit.log file that uses all of lv_audit partition. Doubling its size to avoid having LowDiskSpace from SCT. --- .../ci-aio/inventory/group_vars/controllers/lvm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml index 2c9a52142..9e337ebc4 100644 --- a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml +++ b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml @@ -25,7 +25,7 @@ stackhpc_lvm_lv_var_tmp_size: 2g stackhpc_lvm_lv_log_size: 2g # StackHPC LVM lv_audit LV size. -stackhpc_lvm_lv_audit_size: 250m +stackhpc_lvm_lv_audit_size: 500m # StackHPC LVM lv_home LV size. stackhpc_lvm_lv_home_size: 2g From 9751360ac094b3794d4399e0952971a01bb03ce3 Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 9 Jun 2025 09:48:32 +0100 Subject: [PATCH 4/4] Bump upgrading AIO volume to 75 GB --- .github/workflows/stackhpc-all-in-one.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 7674ba849..a1c2e78ed 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -176,7 +176,7 @@ jobs: VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}" VM_IMAGE: ${{ steps.image_name.outputs.image_name }} VM_INTERFACE: ${{ inputs.vm_interface }} - VM_VOLUME_SIZE: ${{ inputs.upgrade && '65' || '50' }} + VM_VOLUME_SIZE: ${{ inputs.upgrade && '75' || '50' }} VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]' - name: Terraform Plan