Sync stackhpc/2024.1 into master

.github/workflows/stackhpc-all-in-one.yml

@@ -423,7 +423,8 @@ jobs:
             -v $(pwd)/sct-results:/stack/sct-results \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
-            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-cloud-tests.yml' -e sct_version=${{ inputs.stackhpc_cloud_tests_version }}
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-cloud-tests.yml' \
+            -e sct_version=${{ inputs.stackhpc_cloud_tests_version }}
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 

.github/workflows/stackhpc-container-image-build.yml

@@ -92,17 +92,17 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          comma=""
-          echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
+          output="{'distro': ["
           if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
-            echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
-            comma=", "
+              output+="{'name': 'rocky', 'release': 9},"
           fi
           if [[ ${{ inputs.ubuntu-noble }} == 'true' ]]; then
-            echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
-            comma=", "
+              output+="{'name': 'ubuntu', 'release': 'noble'},"
           fi
-          echo "]}" >> $GITHUB_OUTPUT
+          # remove trailing comma
+          output="${output%,}"
+          output+="]}"
+          echo "matrix=$output" >> $GITHUB_OUTPUT
 
       - name: Display container datetime tag
         run: |
@@ -173,7 +173,7 @@ jobs:
 
       - name: Get Kolla tag
         id: write-kolla-tag
-        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'noble' }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
+        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
 
       - name: Configure localhost as a seed
         run: |
@@ -203,7 +203,8 @@ jobs:
         continue-on-error: true
         run: |
           args="${{ inputs.regexes }}"
-          args="$args -e kolla_base_distro=${{ matrix.distro }}"
+          args="$args -e kolla_base_distro=${{ matrix.distro.name }}"
+          args="$args -e kolla_base_distro_version=${{ matrix.distro.release }}"
           args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
@@ -221,7 +222,8 @@ jobs:
         id: build_seed_images
         continue-on-error: true
         run: |
-          args="-e kolla_base_distro=${{ matrix.distro }}"
+          args="-e kolla_base_distro=${{ matrix.distro.name }}"
+          args="$args -e kolla_base_distro_version=${{ matrix.distro.release }}"
           args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
@@ -236,13 +238,13 @@ jobs:
         if: inputs.seed
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images
 
       - name: Fail if no images have been built
-        run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
+        run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Scan built container images
-        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}
 
       - name: Move image scan logs to output artifact
         run: mv image-scan-output image-build-logs/image-scan-output
@@ -297,7 +299,7 @@ jobs:
       - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }}-logs
+          name: ${{ matrix.distro.name }}-${{ matrix.distro.release }}-logs
           path: image-build-logs
           retention-days: 7
         if: ${{ !cancelled() }}

.github/workflows/stackhpc-pull-request.yml

@@ -18,6 +18,9 @@ jobs:
       pull-requests: read
     name: Check changed files
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    needs:
+      - lint
+      - tox
     outputs:
       aio: ${{ steps.changes.outputs.aio }}
       build-kayobe-image: ${{ steps.changes.outputs.build-kayobe-image }}

.github/workflows/stackhpc-update-kolla.yml

@@ -0,0 +1,29 @@
+name: Update Kolla versions
+
+on:
+  # Allow manual executions
+  workflow_dispatch:
+  # Run weekly on Tuesday
+  schedule:
+    - cron: '0 0 * * 2'
+
+jobs:
+  update-from-branch:
+    name: Update dependencies
+    strategy:
+      matrix:
+        include:
+          - version: stackhpc/2023.1
+            codename: Antelope
+          - version: stackhpc/2024.1
+            codename: Caracal
+          - version: stackhpc/master
+            codename: master
+    uses: ./.github/workflows/update-dependencies.yml
+    with:
+      openstack_version: ${{ matrix.version }}
+      openstack_codename: ${{ matrix.codename }}
+    permissions:
+      contents: write
+      pull-requests: write
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'

.github/workflows/update-dependencies.yml

@@ -1,11 +1,16 @@
 name: Update dependencies
 
 on:
-  # Allow manual executions
-  workflow_dispatch:
-  # Run nightly
-  schedule:
-    - cron: '0 0 * * *'
+  workflow_call:
+    inputs:
+      openstack_version:
+        description: OpenStack version
+        type: string
+        required: true
+      openstack_codename:
+        description: OpenStack codename
+        type: string
+        required: true
 
 jobs:
   propose_github_release_updates:
@@ -38,20 +43,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          ref: ${{ inputs.openstack_version }}
           path: ${{ github.workspace }}/src/kayobe-config
 
-      - name: Determine OpenStack release
-        id: openstack_release
-        run: |
-          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
-          echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
-        working-directory: ${{ github.workspace }}/src/kayobe-config
-
       - name: Checkout the dependency repo
         uses: actions/checkout@v4
         with:
           repository: ${{ matrix.repository }}
-          ref: stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
+          ref: ${{ inputs.openstack_version }}
           fetch-tags: true
           path: ${{ github.workspace }}/src/${{ matrix.key }}
 
@@ -78,10 +77,17 @@ jobs:
           path: ${{ github.workspace }}/src/kayobe-config
           commit-message: >-
             Bump ${{ matrix.key }} to ${{ steps.latest_tag.outputs.latest_tag }}
-          branch: update-dependency/${{ matrix.key }}
+          author: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com>
+          branch: update-dependency/${{ matrix.key }}/${{ inputs.openstack_version }}
           delete-branch: true
           title: >-
             Bump ${{ matrix.key }} to ${{ steps.latest_tag.outputs.latest_tag }}
           body: >
-            This PR was created automatically to update
+            This PR was created automatically to update ${{ inputs.openstack_version }}
             ${{ matrix.key }} to ${{ steps.latest_tag.outputs.latest_tag }}.
+
+            GitHub Release Changelog:
+            https://github.com/stackhpc/${{ matrix.key }}/releases/tag/${{ steps.latest_tag.outputs.latest_tag }}
+          labels: |
+            automated
+            ${{ inputs.openstack_codename }}

doc/source/configuration/cephadm.rst

@@ -206,8 +206,8 @@ Pools
 ~~~~~
 
 The following example pools should be sufficient to work with the
-default `external Ceph
-configuration <https://docs.openstack.org/kolla-ansible/latest/reference/storage/external-ceph-guide.html>`__
+default :kolla-ansible-doc:`external Ceph
+configuration <reference/storage/external-ceph-guide.html>`
 for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible.
 
 .. code:: yaml
@@ -234,8 +234,8 @@ Keys
 ~~~~
 
 The following example keys should be sufficient to work with the default
-`external Ceph
-configuration <https://docs.openstack.org/kolla-ansible/latest/reference/storage/external-ceph-guide.html>`__
+:kolla-ansible-doc:`external Ceph
+configuration <reference/storage/external-ceph-guide.html>`
 for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible.
 
 .. code:: yaml
@@ -351,7 +351,7 @@ RADOS Gateways
 
 RADOS Gateway integration is described in the :kolla-ansible-doc:`Kolla Ansible
 documentation
-<https://docs.openstack.org/kolla-ansible/latest/reference/storage/external-ceph-guide.html#radosgw>`.
+<reference/storage/external-ceph-guide.html#radosgw>`.
 
 RADOS Gateways (RGWs) are defined with the following:
 
@@ -390,7 +390,7 @@ The set of commands below configure all of these.
   - "config set client.rgw rgw_keystone_admin_user 'ceph_rgw'"
   - "config set client.rgw rgw_keystone_api_version '3'"
   - "config set client.rgw rgw_keystone_token_cache_size '10000'"
-  - "config set client.rgw rgw_keystone_url https://{{ kolla_internal_fqdn }}:5000"
+  - "config set client.rgw rgw_keystone_url {{ 'https' if kolla_enable_tls_internal | bool else 'http' }}://{{ kolla_internal_fqdn }}:5000"
   - "config set client.rgw rgw_keystone_verify_ssl false"
   - "config set client.rgw rgw_max_attr_name_len '1000'"
   - "config set client.rgw rgw_max_attr_size '1000'"

doc/source/configuration/ci-cd.rst

@@ -118,7 +118,7 @@ Runner Deployment
     The repository permissions for a fine-grained personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R`
     Once the key has been obtained, add it to :code:`secrets.yml` under :code:`secrets_github_access_token`
 
-7. If the host is an actual Infra VM then please refer to upstream `Infrastructure VMs <https://docs.openstack.org/kayobe/latest/configuration/reference/infra-vms.html>`__ documentation for additional configuration and steps.
+7. If the host is an actual Infra VM then please refer to upstream :kayobe-doc:`Infrastructure VMs <configuration/reference/infra-vms.html>` documentation for additional configuration and steps.
 
 8. Run :code:`kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-github-runner.yml`
 

doc/source/configuration/cloudkitty.rst

@@ -8,8 +8,8 @@ Configuring in kayobe-config
 By default, CloudKitty uses Gnocchi and Ceilometer as the collector and fetcher
 backends. Unless the system has a specific reason not to, we recommend instead
 using Prometheus as the backend for both. The following instructions explain
-how to do this. Also, see the `Kolla Ansible docs on CloudKitty
-<https://docs.openstack.org/kolla-ansible/latest/reference/rating/cloudkitty-guide.html>`__
+how to do this. Also, see the :kolla-ansible-doc:`Kolla Ansible docs on CloudKitty
+<reference/rating/cloudkitty-guide.html>`
 for more details.
 
 Enable CloudKitty and disable InfluxDB, as we are using OpenSearch as the

doc/source/configuration/index.rst

@@ -13,6 +13,7 @@ the various features provided.
    lvm
    cephadm
    monitoring
+   openbao
    vault
    wazuh
    walled-garden

doc/source/configuration/monitoring.rst

@@ -81,8 +81,8 @@ on the overcloud hosts:
 SMART reporting should now be enabled along with a Prometheus alert for
 unhealthy disks and a Grafana dashboard called ``Hardware Overview``.
 
-Alertmanager and Slack
-======================
+Alertmanager, Slack and Microsoft Teams
+=======================================
 
 StackHPC Kayobe configuration comes bundled with an array of alerts but does not
 enable any receivers for notifications by default. Various receivers can be
@@ -119,6 +119,17 @@ available `here <https://awesome-prometheus-alerts.grep.to/>`__. They simply
 need to be added to one of the ``*.rules`` files in the prometheus configuration
 directory.
 
+If however you are using Microsoft Teams instead of Slack, you can use Prometheus
+Alertmanager's built-in support for the new message format based on Power Automate flows.
+You will need an incoming webhook URL for your Teams channel.
+This can be done by following `these instructions <https://support.microsoft.com/en-gb/office/create-incoming-webhooks-with-workflows-for-microsoft-teams-8ae491c7-0394-4861-ba59-055e33f75498>`__.
+To set up a receiver, create a ``prometheus-alertmanager.yml`` file under
+``etc/kayobe/kolla/config/prometheus/``.
+An example config is stored in this directory known as ``prometheus-alertmanager.msteamvs2.yml.example``.
+The example configuration uses two Slack channels.
+One channel receives all alerts while the other only receives alerts tagged as critical.
+Feel free to modify the example configuration to suit your needs.
+
 Ceph Monitoring
 ===============