wip

Author: Alex-Welsh

.github/workflows/stackhpc-all-in-one.yml

@@ -344,6 +344,17 @@ jobs:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
         if: inputs.upgrade
 
+      - name: Migrate RabbitMQ queues
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            ${{ steps.kayobe_image.outputs.kayobe_image }} \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/migrate-rabbitmq-queues.yml --tags rabbit-queue-migration
+        env:
+          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
+        if: inputs.upgrade
+
       - name: Service upgrade
         run: |
           docker run -t --rm \

etc/kayobe/ansible/migrate-rabbitmq-queues.yml

@@ -0,0 +1,84 @@
+---
+# Migrate RabbitMQ queues from HA to Quorum
+# This is primarily used in CI workflows
+
+- name: Migrate RabbitMQ queues
+  hosts: localhost
+  gather_facts: no
+  vars:
+    - rabbitmq_container_name: "rabbitmq"
+    - services_to_restart: "barbican,blazar,cinder,cloudkitty,designate,heat,ironic,keystone,magnum,manila,neutron,nova,octavia"
+  tasks:
+    - name: Checking timedatectl status
+      command: timedatectl status
+      register: timedatectl_status
+      changed_when: false
+
+    - name: Fail if the clock is not synchronized
+      assert:
+        that:
+          - "'synchronized: yes' not in timedatectl_status.stdout"
+        fail_msg: >
+          timedatectl sees the system clock as unsynchronized.
+          You may need to force synchronisation using `chronyc makestep`.
+          Otherwise, please wait for synchronization.
+
+    - name: Inspect the {{ rabbitmq_container_name }} container
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
+      register: inspection
+      become: true
+
+    - name: Ensure the {{ rabbitmq_container_name }} container is running
+      command: "systemctl start kolla-{{ rabbitmq_container_name }}-container.service"
+      when: inspection.stdout == 'false'
+      become: true
+
+    - name: Wait for the {{ rabbitmq_container_name }} container to reach state 'Running'
+      shell:
+        cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
+      register: result
+      until: result.stdout == 'true'
+      retries: 10
+      delay: 6
+      become: true
+
+    - name: Wait for the rabbitmq node to automatically start on container start
+      command: "docker exec {{ rabbitmq_container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60'"
+      when: inspection.stdout == 'false'
+      become: true
+
+    - name: Migrate RabbitMQ queues
+      shell:
+        cmd: >
+          kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
+          kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt {{ services_to_restart }} -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
+          kayobe kolla ansible run rabbitmq-reset-state -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
+          kayobe kolla ansible run deploy -kt {{ services_to_restart }} -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false
+        executable: /bin/bash
+      run_once: true
+      delegate_to: localhost
+      tags: rabbit-queue-migration
+
+    - name: Inspect RabbitMQ queues
+      shell:
+        cmd: "docker exec {{ rabbitmq_container_name }} rabbitmqctl list_queues type"
+      run_once: true
+      delegate_to: localhost
+      register: queues
+      become: true
+
+    - name: Assert that queues have been migrated
+      assert:
+        that: "{{ 'quorum' in queues.stdout }}"
+        fail_msg: Queue migration has failed. Run the migration manually.
+      run_once: true
+      delegate_to: localhost
+
+    - name: Print warning message
+      debug:
+        msg: >
+          Queues have been migrated, now ensure
+          om_enable_rabbitmq_quorum_queues is set to true in kolla/globals.yml
+      run_once: true
+      delegate_to: localhost

etc/kayobe/ansible/prometheus.yml.j2

@@ -247,18 +247,6 @@ scrape_configs:
 {% endfor %}
 {% endif %}
 
-{% if enable_ironic_prometheus_exporter | bool %}
-  - job_name: ironic_prometheus_exporter
-    static_configs:
-{% for host in groups['ironic-conductor'] %}
-      - targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"]
-{% if hostvars[host].prometheus_instance_label | default(false, true) %}
-        labels:
-          instance: "{{ hostvars[host].prometheus_instance_label }}"
-{% endif %}
-{% endfor %}
-{% endif %}
-
 {% if enable_prometheus_alertmanager | bool %}
   - job_name: alertmanager
     static_configs:
@@ -273,14 +261,10 @@ scrape_configs:
 
 alerting:
   alertmanagers:
-  - static_configs:
-    - targets:
+    - static_configs:
 {% for host in groups["prometheus-alertmanager"] %}
+      - targets:
         - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
-{% if hostvars[host].prometheus_instance_label | default(false, true) %}
-        labels:
-          instance: "{{ hostvars[host].prometheus_instance_label }}"
-{% endif %}
 {% endfor %}
 {% endif %}
 {% endraw %}

etc/kayobe/kolla-image-tags.yml

@@ -6,3 +6,6 @@ kolla_image_tags:
   openstack:
     rocky-9: 2024.1-rocky-9-20240725T165045
     ubuntu-jammy: 2024.1-ubuntu-jammy-20240725T165045
+  heat:
+    rocky-9: 2024.1-rocky-9-20240805T142526
+    ubuntu-jammy: 2024.1-ubuntu-jammy-20240805T142526

etc/kayobe/stackhpc.yml

@@ -145,7 +145,8 @@ stackhpc_kolla_source_version: "stackhpc/{{ openstack_release }}"
 
 # Kolla Ansible source repository.
 stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"
-stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
+# stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
+stackhpc_kolla_ansible_source_version: "fix-prometheus"
 
 ###############################################################################
 # Container image registry

terraform/aio/vm.tf

@@ -93,6 +93,7 @@ resource "null_resource" "kayobe-aio" {
     inline = [
       "#!/bin/sh",
       "echo 'connected!'"
+      "sudo hostnamectl set-hostname $(sed s/.novalocal// /etc/hostname)"
       ]
   }
 }