Merge branch 'stackhpc/master' into master-requirements

Author: Alex-Welsh

.ansible-lint-ignore

@@ -5,3 +5,5 @@ etc/kayobe/ansible/vault-generate-internal-tls.yml fqcn[action-core]
 etc/kayobe/ansible/vault-generate-test-external-tls.yml fqcn[action-core]
 etc/kayobe/ansible/rabbitmq-reset.yml command-instead-of-module
 etc/kayobe/ansible/ubuntu-upgrade.yml syntax-check[missing-file]
+etc/kayobe/ansible/check-kayobe-version.yml command-instead-of-module
+etc/kayobe/ansible/check-kolla-ansible-version.yml command-instead-of-module

.github/workflows/runner-selector.yml

@@ -37,7 +37,7 @@ jobs:
 
       - name: Set output for container image build runner
         run: echo "Setting runner for ${{ inputs.runner_env }} -> ${{ vars.RUNS_ON_TARGET_CONTAINER_IMAGE_BUILDER }}"
-        
+
       - id: container-image-build-runner
         run: echo "runner_name_container_image_build=${{ vars.RUNS_ON_TARGET_CONTAINER_IMAGE_BUILDER }}" >> $GITHUB_OUTPUT
 

.github/workflows/stackhpc-all-in-one.yml

@@ -7,11 +7,10 @@ name: All in one
 on:
   workflow_call:
     inputs:
-      runner:
-        required: false
+      runner_env:
+        description: Which cloud to run on?
         type: string
-        description: 'Runner name'
-        default: 'arc-skc-aio-runner'
+        default: SMS Lab
       kayobe_image:
         description: Kayobe container image
         type: string
@@ -40,18 +39,6 @@ on:
         description: Default network interface name
         type: string
         default: ens3
-      vm_flavor:
-        description: Flavor for the all-in-one VM
-        type: string
-        default: en1.medium
-      vm_network:
-        description: Network for the all-in-one VM
-        type: string
-        default: stackhpc-ci
-      vm_subnet:
-        description: Subnet for the all-in-one VM
-        type: string
-        default: stackhpc-ci
       OS_CLOUD:
         description: Name of cloud in clouds.yaml
         type: string
@@ -87,11 +74,18 @@ on:
         required: true
 
 jobs:
+  runner-selection:
+    uses: ./.github/workflows/runner-selector.yml
+    with:
+      runner_env: ${{ inputs.upgrade == true && 'Leafcloud' || inputs.runner_env }}
   # NOTE: Runner needs unzip and nodejs packages.
   all-in-one:
     name: All in one
     if: ${{ inputs.if && !cancelled() }}
-    runs-on: ${{ inputs.runner }}
+    environment: ${{ inputs.upgrade == true && 'Leafcloud' || inputs.runner_env }}
+    runs-on: ${{ needs.runner-selection.outputs.runner_name_aio }}
+    needs:
+      - runner-selection
     permissions: {}
     env:
       KAYOBE_ENVIRONMENT: ci-aio
@@ -170,9 +164,9 @@ jobs:
           aio_vm_interface = "${{ env.VM_INTERFACE }}"
           aio_vm_name = "${{ env.VM_NAME }}"
           aio_vm_image = "${{ env.VM_IMAGE }}"
-          aio_vm_flavor = "${{ env.VM_FLAVOR }}"
-          aio_vm_network = "${{ env.VM_NETWORK }}"
-          aio_vm_subnet = "${{ env.VM_SUBNET }}"
+          aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}"
+          aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}"
+          aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}"
           aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
           aio_vm_tags = ${{ env.VM_TAGS }}
           EOF
@@ -181,9 +175,6 @@ jobs:
           SSH_USERNAME: "${{ inputs.ssh_username }}"
           VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
           VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
-          VM_FLAVOR: ${{ inputs.vm_flavor }}
-          VM_NETWORK: ${{ inputs.vm_network }}
-          VM_SUBNET: ${{ inputs.vm_subnet }}
           VM_INTERFACE: ${{ inputs.vm_interface }}
           VM_VOLUME_SIZE: ${{ inputs.upgrade && '65' || '50' }}
           VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
@@ -192,7 +183,7 @@ jobs:
         run: terraform plan
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
-          OS_CLOUD: ${{ inputs.OS_CLOUD }}
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
@@ -213,7 +204,7 @@ jobs:
           exit 1
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
-          OS_CLOUD: ${{ inputs.OS_CLOUD }}
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
@@ -471,7 +462,7 @@ jobs:
         run: terraform destroy -auto-approve
         working-directory: ${{ github.workspace }}/terraform/aio
         env:
-          OS_CLOUD: ${{ inputs.OS_CLOUD }}
+          OS_CLOUD: ${{ vars.OS_CLOUD }}
           OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
           OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
         if: always()

.github/workflows/stackhpc-pull-request.yml

@@ -236,3 +236,21 @@ jobs:
       upgrade: true
     secrets: inherit
     if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}
+
+  all-in-one-upgrade-rocky-9-ovs:
+    name: aio upgrade (Rocky 9 OVS)
+    needs:
+      - check-changes
+      - build-kayobe-image
+    uses: ./.github/workflows/stackhpc-all-in-one.yml
+    with:
+      kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }}
+      os_distribution: rocky
+      os_release: "9"
+      ssh_username: cloud-user
+      neutron_plugin: ovs
+      OS_CLOUD: openstack
+      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      upgrade: true
+    secrets: inherit
+    if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}

.github/workflows/update-dependencies.yml

@@ -14,6 +14,7 @@ on:
 
 jobs:
   propose_github_release_updates:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: ubuntu-22.04
     strategy:
       matrix:

.github/workflows/upstream-sync.yml

@@ -0,0 +1,38 @@
+---
+name: Upstream Sync
+'on':
+  schedule:
+    - cron: "15 8 * * 1"
+  workflow_dispatch:
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  synchronise-2023-1:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise 2023.1
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: 2023.1
+      upstream: openstack/kayobe-config
+  synchronise-2024-1:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise 2024.1
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: 2024.1
+      upstream: openstack/kayobe-config
+  synchronise-2025-1:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise 2025.1
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: 2025.1
+      upstream: openstack/kayobe-config
+  synchronise-master:
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
+    name: Synchronise master
+    uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main
+    with:
+      release_series: master
+      upstream: openstack/kayobe-config

doc/source/configuration/ipa.rst

@@ -11,7 +11,7 @@ StackHPC provides prebuilt Ironic Python Agent (IPA) images in Release Train
 through Ark.
 
 These images are built in CI using a GitHub workflow and are configured in this
-repository. See  :kayobe-doc: `Kayobe documentation
+repository. See :kayobe-doc:`Kayobe documentation
 <configuration/reference/ironic-python-agent.html>` for more details on IPA.
 
 Release Train IPA images are used by Bifrost and Overcloud Ironic by default in

doc/source/configuration/release-train.rst

@@ -52,16 +52,29 @@ The Pulp container is deployed on the seed by default, but may be disabled by
 setting ``seed_pulp_container_enabled`` to ``false`` in
 ``etc/kayobe/seed.yml``.
 
-The URL and credentials of the local Pulp server are configured in
-``etc/kayobe/pulp.yml`` via ``pulp_url``, ``pulp_username`` and
-``pulp_password``. In most cases, the default values should be sufficient.
-An admin password must be generated and set as the value of a
-``secrets_pulp_password`` variable, typically in an Ansible Vault encrypted
-``etc/kayobe/secrets.yml`` file. This password will be automatically set on
-Pulp startup.
-
-If a proxy is required to access the Internet from the seed, ``pulp_proxy_url``
-may be used.
+The URL for the local Pulp server is configured by ``pulp_url`` within
+``etc/kayobe/pulp.yml``.
+
+The Pulp service can be configured with two sets of credentials; one for
+administrator operations and another read-only for overcloud hosts
+to use.
+The administrator credentials can be configured ``pulp_username``,
+``pulp_password``
+The basic user account credentials can be configured with ``pulp_stack_username``
+and ``pulp_stack_password``.
+Both sets of credentials can be found within ``etc/kayobe/pulp.yml``.
+
+Both the ``pulp_password`` and ``pulp_stack_password`` are intended to be
+configured via their ``secrets_*`` counterparts, i.e.
+``secrets_pulp_password`` and ``secrets_pulp_stack_password``. These variables
+are expected to be set in an Ansible Vault encrypted
+``etc/kayobe/secrets.yml`` file.
+
+Passwords can be generated using ``OpenSSL``
+
+.. code-block:: console
+
+  openssl rand -base64 32
 
 Host images are not synchronised to the local Pulp server, since they should
 only be pulled to the seed node once. More information on host images can be

doc/source/operations/upgrading-openstack.rst

@@ -106,6 +106,39 @@ the following in ``kayobe-config/etc/kayobe/stackhpc-monitoring.yml``:
    # targets being templated during deployment.
    stackhpc_enable_os_capacity: false
 
+Prometheus blackbox exporter endpoints
+--------------------------------------
+
+Many endpoints for the Blackbox exporter are now templated in the Kolla-Ansible
+group vars for the cloud. This means that the
+``prometheus_blackbox_exporter_endpoints`` variable can be removed from the
+environment's ``kolla/globals.yml`` file (if applicable) and the endpoints will
+fallback to the ones templated in the group vars. Backend endpoints such as
+`these <https://github.com/stackhpc/stackhpc-kayobe-config/blob/094c2e012a037309d103c08a71eb633fdeb214e7/etc/kayobe/kolla/inventory/group_vars/prometheus-blackbox-exporter#L27-L64>`__
+are not yet templated by Kolla-Ansible.
+
+Additional endpoints may still be added.
+
+For Kolla-Ansible templating, use ``stackhpc_prometheus_blackbox_exporter_endpoints_custom``.
+For example:
+
+.. code-block:: yaml
+   :caption: ``etc/kayobe/kolla/inventory/group_vars/prometheus-blackbox-exporter``
+
+   stackhpc_prometheus_blackbox_exporter_endpoints_custom:
+     - 'custom_service:http_2xx:{{ public_protocol }}://{{ external_fqdn | put_address_in_context('url') }}:{{ custom_serivce_port }}'
+
+Alternatively, for Kayobe templating, use the ``prometheus_blackbox_exporter_endpoints_kayobe`` variable.
+For example:
+
+.. code-block:: yaml
+   :caption: ``kolla/globals.yml``
+
+   prometheus_blackbox_exporter_endpoints_kayobe:
+      - endpoints:
+         - "pulp:http_2xx:{{ pulp_url }}/pulp/api/v3/status/"
+      enabled: "{{ seed_pulp_container_enabled | bool }}"
+
 Known issues
 ============
 

etc/kayobe/ansible/cephadm-gather-keys.yml

@@ -68,6 +68,7 @@
         # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf.
         content: |
           {{ cephadm_ceph_conf.stdout | regex_replace('\t') }}
+          {{ kolla_ceph_conf_append if kolla_ceph_conf_append is defined }}
         dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf"
       loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}"
       loop_control: