Skip to content

Commit cb554f4

Browse files
Alex-WelshAlex-Welsh
committed
wip
1 parent b186ebd commit cb554f4

File tree

5 files changed

+100
-19
lines changed

5 files changed

+100
-19
lines changed

‎.github/workflows/stackhpc-all-in-one.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,17 @@ jobs:
344344
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
345345
if: inputs.upgrade
346346

347+
- name: Migrate RabbitMQ queues
348+
run: |
349+
docker run -t --rm \
350+
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
351+
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
352+
${{ steps.kayobe_image.outputs.kayobe_image }} \
353+
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/migrate-rabbitmq-queues.yml
354+
env:
355+
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
356+
if: inputs.upgrade
357+
347358
- name: Service upgrade
348359
run: |
349360
docker run -t --rm \

‎etc/kayobe/ansible/migrate-rabbitmq-queues.yml

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
---
2+
# Migrate RabbitMQ queues from HA to Quorum
3+
4+
- name: Migrate RabbitMQ queues
5+
hosts: localhost
6+
gather_facts: no
7+
vars:
8+
- rabbitmq_container_name: "rabbitmq"
9+
- services_to_restart: "barbican,blazar,cinder,cloudkitty,designate,heat,ironic,keystone,magnum,manila,neutron,nova,octavia"
10+
tasks:
11+
- name: Checking timedatectl status
12+
command: timedatectl status
13+
register: timedatectl_status
14+
changed_when: false
15+
16+
- name: Fail if the clock is not synchronized
17+
assert:
18+
that:
19+
- "'synchronized: yes' not in timedatectl_status.stdout"
20+
fail_msg: >
21+
timedatectl sees the system clock as unsynchronized.
22+
You may need to force synchronisation using `chronyc makestep`.
23+
Otherwise, please wait for synchronization.
24+
25+
- name: Inspect the {{ rabbitmq_container_name }} container
26+
shell:
27+
cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
28+
register: inspection
29+
become: true
30+
31+
- name: Ensure the {{ rabbitmq_container_name }} container is running
32+
command: "systemctl start kolla-{{ rabbitmq_container_name }}-container.service"
33+
when: inspection.stdout == 'false'
34+
become: true
35+
36+
- name: Wait for the {{ rabbitmq_container_name }} container to reach state 'Running'
37+
shell:
38+
cmd: "docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ rabbitmq_container_name }}"
39+
register: result
40+
until: result.stdout == 'true'
41+
retries: 10
42+
delay: 6
43+
become: true
44+
45+
- name: Wait for the rabbitmq node to automatically start on container start
46+
command: "docker exec {{ rabbitmq_container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60'"
47+
when: inspection.stdout == 'false'
48+
become: true
49+
50+
- name: Migrate RabbitMQ queues
51+
shell:
52+
cmd: >
53+
kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
54+
kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt {{ services_to_restart }} -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
55+
kayobe kolla ansible run rabbitmq-reset-state -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false &&
56+
kayobe kolla ansible run deploy -kt {{ services_to_restart }} -ke om_enable_rabbitmq_quorum_queues=true -ke om_enable_rabbitmq_high_availability=false
57+
executable: /bin/bash
58+
run_once: true
59+
delegate_to: localhost
60+
61+
- name: Inspect RabbitMQ queues
62+
shell:
63+
cmd: "docker exec {{ rabbitmq_container_name }} rabbitmqctl list_queues type"
64+
run_once: true
65+
delegate_to: localhost
66+
register: queues
67+
become: true
68+
69+
- name: Assert that queues have been migrated
70+
assert:
71+
that: "{{ 'quorum' in queues.stdout }}"
72+
fail_msg: Queue migration has failed. Run the migration manually.
73+
run_once: true
74+
delegate_to: localhost
75+
76+
- name: Print warning message
77+
debug:
78+
msg: >
79+
Queues have been migrated, now ensure
80+
om_enable_rabbitmq_quorum_queues is set to true in kolla/globals.yml
81+
run_once: true
82+
delegate_to: localhost

‎etc/kayobe/ansible/prometheus.yml.j2

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -247,18 +247,6 @@ scrape_configs:
247247
{% endfor %}
248248
{% endif %}
249249

250-
{% if enable_ironic_prometheus_exporter | bool %}
251-
- job_name: ironic_prometheus_exporter
252-
static_configs:
253-
{% for host in groups['ironic-conductor'] %}
254-
- targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"]
255-
{% if hostvars[host].prometheus_instance_label | default(false, true) %}
256-
labels:
257-
instance: "{{ hostvars[host].prometheus_instance_label }}"
258-
{% endif %}
259-
{% endfor %}
260-
{% endif %}
261-
262250
{% if enable_prometheus_alertmanager | bool %}
263251
- job_name: alertmanager
264252
static_configs:
@@ -273,14 +261,10 @@ scrape_configs:
273261

274262
alerting:
275263
alertmanagers:
276-
- static_configs:
277-
- targets:
264+
- static_configs:
278265
{% for host in groups["prometheus-alertmanager"] %}
266+
- targets:
279267
- '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
280-
{% if hostvars[host].prometheus_instance_label | default(false, true) %}
281-
labels:
282-
instance: "{{ hostvars[host].prometheus_instance_label }}"
283-
{% endif %}
284268
{% endfor %}
285269
{% endif %}
286270
{% endraw %}

‎etc/kayobe/kolla-image-tags.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ kolla_image_tags:
66
openstack:
77
rocky-9: 2024.1-rocky-9-20240725T165045
88
ubuntu-jammy: 2024.1-ubuntu-jammy-20240725T165045
9+
heat:
10+
rocky-9: 2024.1-rocky-9-20240805T142526
11+
ubuntu-jammy: 2024.1-ubuntu-jammy-20240805T142526

‎etc/kayobe/stackhpc.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,8 @@ stackhpc_kolla_source_version: "stackhpc/{{ openstack_release }}"
145145

146146
# Kolla Ansible source repository.
147147
stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible"
148-
stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
148+
# stackhpc_kolla_ansible_source_version: "stackhpc/{{ openstack_release }}"
149+
stackhpc_kolla_ansible_source_version: "fix-prometheus"
149150

150151
###############################################################################
151152
# Container image registry

0 commit comments

Comments
 (0)