From c3acfa8cf65fc372abea5ce2b6de57ef1df8a3f4 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 10 Jun 2025 15:03:16 +0100 Subject: [PATCH 1/3] Put the RadosGW Usage Exporter behind HAProxy --- .../ansible/deploy-radosgw-usage-exporter.yml | 2 +- etc/kayobe/kolla.yml | 2 ++ .../services.d/radosgw_usage_exporter.cfg | 25 +++++++++++++++++++ .../prometheus.yml.d/80-radosgw-exporter.yml | 7 +++--- etc/kayobe/stackhpc-monitoring.yml | 7 ++++-- 5 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg diff --git a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml index e7c0cf254..d9db106bf 100644 --- a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml +++ b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml @@ -114,7 +114,7 @@ ADMIN_ENTRY: admin ACCESS_KEY: "{{ ec2.Access }}" SECRET_KEY: "{{ ec2.Secret }}" - VIRTUAL_PORT: "{{ stackhpc_radosgw_usage_exporter_port | string }}" + VIRTUAL_PORT: "{{ stackhpc_radosgw_usage_exporter_backend_port | string }}" REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" entrypoint: "{{ ['python', '-u', './radosgw_usage_exporter.py', '--insecure'] if not stackhpc_radosgw_usage_exporter_verify else omit }}" vars: diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5eb6740dc..f5f106c40 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -560,6 +560,8 @@ kolla_build_args: {} kolla_overcloud_inventory_pass_through_host_vars_extra: - stackhpc_gpu_data - gpu_group_map + - stackhpc_radosgw_usage_exporter_frontend_port + - stackhpc_radosgw_usage_exporter_backend_port # List of names of host variables to pass through from kayobe hosts to # kolla-ansible hosts, if set. See also diff --git a/etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg b/etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg new file mode 100644 index 000000000..50e55a283 --- /dev/null +++ b/etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg @@ -0,0 +1,25 @@ +{% if stackhpc_enable_radosgw_usage_exporter | bool %} +{% raw %} +frontend radosgw_usage_exporter_frontend + mode http + http-request del-header X-Forwarded-Proto + option httplog + option forwardfor + http-request set-header X-Forwarded-Proto https if { ssl_fc } +{% if kolla_enable_tls_internal | bool %} + bind {{ kolla_internal_vip_address }}:{{ stackhpc_radosgw_usage_exporter_frontend_port }} ssl crt /etc/haproxy/certificates/haproxy-internal.pem +{% else %} + bind {{ kolla_internal_vip_address }}:{{ stackhpc_radosgw_usage_exporter_frontend_port }} +{% endif %} + default_backend radosgw_usage_exporter_backend + +backend radosgw_usage_exporter_backend + mode http + +{% for host in groups['monitoring'] %} +{% set host_name = hostvars[host].ansible_facts.hostname %} +{% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:{{ stackhpc_radosgw_usage_exporter_backend_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% endraw %} +{% endif %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml index 304736a80..7c3a204bd 100644 --- a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml @@ -14,8 +14,9 @@ scrape_configs: regex: (.+) static_configs: - targets: - {% for host in groups['monitoring'] %} - - "{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{% endraw %}{{ stackhpc_radosgw_usage_exporter_port }}{% raw %}" - {% endfor %} + - "{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ stackhpc_radosgw_usage_exporter_frontend_port }}" +{% if kolla_enable_tls_internal | bool %} + scheme: https +{% endif %} {% endraw %} {% endif %} diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml index a2a88b503..f629b153b 100644 --- a/etc/kayobe/stackhpc-monitoring.yml +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -74,8 +74,11 @@ stackhpc_prometheus_openstack_exporter_interval: 300 # Prometheus scrape targets during deployment. stackhpc_enable_radosgw_usage_exporter: false -# Port to expose RADOS gateway usage exporter. Default is 9242 -stackhpc_radosgw_usage_exporter_port: 9242 +# Port to expose RADOS gateway usage exporter backend. Default is 9242 +stackhpc_radosgw_usage_exporter_backend_port: 9242 + +# Port to expose RADOS gateway usage exporter frontend (via HAProxy). Default is 9240 +stackhpc_radosgw_usage_exporter_frontend_port: 9240 # Path to a certificate for internal TLS in the RADOS gateway usage exporter. stackhpc_radosgw_usage_exporter_cacert: "" From d90b7e9334eb06236cb6a7cfc3c778d9c18d5ee4 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 10 Jun 2025 15:53:53 +0100 Subject: [PATCH 2/3] Add reno --- ...dosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml diff --git a/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml b/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml new file mode 100644 index 000000000..66e3a1c6d --- /dev/null +++ b/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue where object storage metrics were missing from Prometheus by + putting the radosgw-usage-exporter behind HAProxy. From 0119d041d5ab46cfccab35a4721b303e93d2ea02 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 11 Jun 2025 16:46:17 +0100 Subject: [PATCH 3/3] Update reno --- ...sgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml b/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml index 66e3a1c6d..338cde866 100644 --- a/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml +++ b/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml @@ -1,4 +1,11 @@ --- +features: + - | + The radosgw-usage-exporter is now put behind HAProxy. To facilitate this, + ``stackhpc_radosgw_usage_exporter_port`` had been renamed to + ``stackhpc_radosgw_usage_exporter_backend_port`` (it remains 9242) and + ``stackhpc_radosgw_usage_exporter_frontend_port`` (defaults to 9240) has + been introduced. fixes: - | Fixes an issue where object storage metrics were missing from Prometheus by