From 759761c41aacdc392f9d28d4e9d73a65d80b7465 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 13 May 2025 22:14:51 +0100 Subject: [PATCH 1/4] feat: add support for `HA` Raft in `OpenBao` --- .../ansible/openbao-deploy-overcloud.yml | 29 ++++++++++++++++++- etc/kayobe/ansible/openbao-deploy-seed.yml | 4 +-- etc/kayobe/ansible/requirements.yml | 4 ++- .../inventory/group_vars/all/openbao.yml | 3 ++ .../add-openbao-raft-ha-e8d78ffe68913512.yaml | 5 ++++ 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml diff --git a/etc/kayobe/ansible/openbao-deploy-overcloud.yml b/etc/kayobe/ansible/openbao-deploy-overcloud.yml index 811e9b1f0..8cf36e57e 100644 --- a/etc/kayobe/ansible/openbao-deploy-overcloud.yml +++ b/etc/kayobe/ansible/openbao-deploy-overcloud.yml @@ -21,7 +21,12 @@ gather_facts: true hosts: controllers vars: - openbao_bind_address: "{{ internal_net_name | net_ip }}" + openbao_bind_addr: "{{ internal_net_name | net_ip }}" + # This is the IP address of the first controller and therefore the leader within + # OpenBao. This could be replaced with the VIP address of the internal network if + # HAProxy has been configured to load balance the OpenBao API. + openbao_raft_leaders: + - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}" tasks: - name: Set a fact about the virtualenv on the remote system ansible.builtin.set_fact: @@ -91,6 +96,28 @@ vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" environment: https_proxy: "" + run_once: true + + # As the first instance is now unsealed the other instances will now need some + # time to connect before we can proceed. + - name: Wait for OpenBao Raft peers to connect + ansible.builtin.wait_for: + timeout: 30 + delegate_to: localhost + + # Raft peers take few seconds before they report an unsealed state therefore + # we must wait. + - name: Unseal OpenBao + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ openbao_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + vault_unseal_timeout: 10 + environment: + https_proxy: "" - name: Configure PKI any_errors_fatal: true diff --git a/etc/kayobe/ansible/openbao-deploy-seed.yml b/etc/kayobe/ansible/openbao-deploy-seed.yml index 2f276cbf6..42972b35e 100644 --- a/etc/kayobe/ansible/openbao-deploy-seed.yml +++ b/etc/kayobe/ansible/openbao-deploy-seed.yml @@ -4,8 +4,8 @@ gather_facts: true hosts: seed vars: - openbao_bind_address: "{{ ansible_facts['lo'].ipv4.address }}" - openbao_api_addr: "http://{{ openbao_bind_address }}:8200" + openbao_bind_addr: "{{ ansible_facts['lo'].ipv4.address }}" + openbao_api_addr: "http://{{ openbao_bind_addr }}:8200" tasks: - name: Set a fact about the virtualenv on the remote system ansible.builtin.set_fact: diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index a81decfc1..8c3adb542 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -9,7 +9,9 @@ collections: - name: stackhpc.pulp version: 0.5.5 - name: stackhpc.hashicorp - version: 2.6.1 + source: https://github.com/stackhpc/ansible-collection-hashicorp + type: git + version: openbao-localhost-fix - name: stackhpc.kayobe_workflows version: 1.1.0 roles: diff --git a/etc/kayobe/inventory/group_vars/all/openbao.yml b/etc/kayobe/inventory/group_vars/all/openbao.yml index 3e084a936..ffd566dc8 100644 --- a/etc/kayobe/inventory/group_vars/all/openbao.yml +++ b/etc/kayobe/inventory/group_vars/all/openbao.yml @@ -77,3 +77,6 @@ seed_openbao_pki_certificate_subject: role: "{{ seed_openbao_pki_role_name }}" extra_params: ip_sans: "{% for host in groups['controllers'] %}{{ internal_net_name | net_ip(host) }}{% if not loop.last %},{% endif %}{% endfor %},{{ kolla_internal_vip_address }}" + +# Enable OpenBao UI +openbao_enable_ui: true diff --git a/releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml b/releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml new file mode 100644 index 000000000..596a3eada --- /dev/null +++ b/releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for highly available Raft when using OpenBao on + overcloud hosts. From b989ef17b1eadd61ba0e94cfbcd96fa2fc66ba7a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 7 Jun 2025 22:09:24 +0100 Subject: [PATCH 2/4] feat: add `openbao_tls_ca` for raft peers --- etc/kayobe/ansible/openbao-deploy-overcloud.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/openbao-deploy-overcloud.yml b/etc/kayobe/ansible/openbao-deploy-overcloud.yml index 8cf36e57e..df3166ce6 100644 --- a/etc/kayobe/ansible/openbao-deploy-overcloud.yml +++ b/etc/kayobe/ansible/openbao-deploy-overcloud.yml @@ -51,7 +51,7 @@ - name: Template out TLS key and cert ansible.builtin.copy: - # Within the OpenBao container these uids & gids map to the vault user + # Within the OpenBao container these uids & gids map to the openbao user src: "{{ kayobe_env_config_path }}/openbao/{{ item }}" dest: /opt/kayobe/openbao/{{ item }} owner: 100 @@ -60,6 +60,7 @@ loop: - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + - "OS-TLS-INT.crt" become: true - name: Apply OpenBao role @@ -76,6 +77,7 @@ openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}" openbao_tls_cert: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" openbao_tls_key: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + openbao_tls_ca: "OS-TLS-INT.crt" copy_self_signed_ca: true openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200 openbao_write_keys_file: true From 8e0593291d523bb3ccc161af8432a4c5fcb75172 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sun, 8 Jun 2025 00:18:14 +0100 Subject: [PATCH 3/4] fix: update CA certificate paths in monitoring and tempest configurations --- doc/source/configuration/monitoring.rst | 8 +++++++- .../environments/ci-multinode/stackhpc-monitoring.yml | 2 +- etc/kayobe/environments/ci-multinode/tempest.yml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 7414a5398..931324e31 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -169,12 +169,18 @@ for the exporter. If you are deploying in a cloud with internal TLS, you may be required to provide a CA certificate for the OpenStack Capacity exporter if your certificate is not signed by a trusted CA. For example, to use a CA certificate -named ``vault.crt`` that is also added to the Kolla containers: +named ``vault.crt`` or ``openbao.crt`` that is also added to the Kolla containers: .. code-block:: yaml stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" +or + +.. code-block:: yaml + + stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" + Alternatively, to disable certificate verification for the OpenStack Capacity exporter: diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml b/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml index 93ce650b4..1d9514553 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml @@ -1,3 +1,3 @@ --- # Path to a CA certificate file to trust in the OpenStack Capacity exporter. -stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" +stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" diff --git a/etc/kayobe/environments/ci-multinode/tempest.yml b/etc/kayobe/environments/ci-multinode/tempest.yml index 0657946bb..ae2d8f132 100644 --- a/etc/kayobe/environments/ci-multinode/tempest.yml +++ b/etc/kayobe/environments/ci-multinode/tempest.yml @@ -3,4 +3,4 @@ rally_no_sensitive_log: false # Add the Vault CA certificate to the rally container when running tempest. -tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" +tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" From 7cc19433e85ee3345d236c566818aa645903c5f4 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Mon, 9 Jun 2025 16:20:34 +0100 Subject: [PATCH 4/4] feat: upgrade `stackhpc.hashicorp` => `2.7.1` --- etc/kayobe/ansible/requirements.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 8c3adb542..7c83c00ec 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -9,9 +9,7 @@ collections: - name: stackhpc.pulp version: 0.5.5 - name: stackhpc.hashicorp - source: https://github.com/stackhpc/ansible-collection-hashicorp - type: git - version: openbao-localhost-fix + version: 2.7.1 - name: stackhpc.kayobe_workflows version: 1.1.0 roles: