Merge pull request #831 from lsst-it/IT-6035_reprovision_kueyen_rke2

(cluster/kueyen) Move Kueyen to RKE2 folder.

Author: gseriche

fleet/lib/kube-prometheus-stack-pre/fleet.yaml

@@ -31,6 +31,7 @@ targetCustomizations:
           values:
             - pillan
             - ruka
+            - kueyen
     yaml:
       overlays:
         - prod

fleet/lib/kube-prometheus-stack/aggregator/values.yaml

@@ -100,7 +100,7 @@ grafana:
       name: $__file{/etc/secrets/keycloak-credentials/url}
       role_attribute_path: contains(groups[*], 'grafana-admin') && 'GrafanaAdmin' || contains(groups[*], 'grafana-admin') && 'Admin' || contains(groups[*], 'grafana-editor') && 'Editor' || 'Viewer'
       scopes: openid profile email groups roles offline_access
-      token_url: https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/token
+      token_url: $__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/token
       use_refresh_token: true
   ingress:
     enabled: true

fleet/lib/kube-prometheus-stack/fleet.yaml

@@ -43,6 +43,19 @@ diff:
       jsonPointers:
         - /spec/automountServiceAccountToken
 targetCustomizations:
+  - name: kueyen
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - kueyen
+    helm:
+      valuesFiles:
+        - pvc/values.yaml
+        - aggregator/values.yaml
+        - ldap/values.yaml
+        - overlays/kueyen/values.yaml
   - name: pillan
     clusterSelector:
       matchExpressions:

fleet/lib/kube-prometheus-stack/overlays/kueyen/values.yaml

@@ -0,0 +1,179 @@
+---
+prometheus:
+  prometheusSpec:
+    configMaps:
+      - sd-snmp-network
+      - sd-snmp-power
+      - sd-snmp-raritan-pdu
+    secrets:
+      - puppetdb
+    additionalScrapeConfigs:
+      - job_name: blackbox-ping-dev
+        metrics_path: /probe
+        params:
+          module: [icmp]
+        puppetdb_sd_configs:
+          - url: https://puppetdb.${ .ClusterLabels.site }.lsst.org:8443
+            basic_auth:
+              username: svc_prometheus
+              password_file: /etc/prometheus/secrets/puppetdb/password
+            query: resources { type = "Class" and title = "Prometheus::Node_exporter" }
+            refresh_interval: 30s
+            follow_redirects: true
+            include_parameters: true
+            enable_http2: true
+        relabel_configs:
+          - source_labels: [__meta_puppetdb_certname]
+            target_label: __param_target
+          - source_labels: [__param_target]
+            target_label: instance
+          - target_label: __address__
+            replacement: prometheus-blackbox-exporter.blackbox-exporter:9115
+      - job_name: node-exporter-dev
+        puppetdb_sd_configs:
+          - url: https://puppetdb.${ .ClusterLabels.site }.lsst.org:8443
+            basic_auth:
+              username: svc_prometheus
+              password_file: /etc/prometheus/secrets/puppetdb/password
+            query: |
+              resources {
+                type = "Class" and title = "Profile::Core::Node_info" and
+                certname in resources[certname] {
+                  type = "Class" and title = "Prometheus::Node_exporter"
+                }
+              }
+            refresh_interval: 30s
+            follow_redirects: true
+            include_parameters: true
+            enable_http2: true
+            port: 9100
+        relabel_configs:
+          - source_labels: [__meta_puppetdb_certname]
+            target_label: instance
+          - source_labels: [__meta_puppetdb_environment]
+            target_label: environment
+          - source_labels: [__meta_puppetdb_parameter_site]
+            target_label: site
+          - source_labels: [__meta_puppetdb_parameter_role]
+            target_label: role
+          - source_labels: [__meta_puppetdb_parameter_cluster]
+            target_label: cluster
+      - job_name: blackbox-ping-ls
+        puppetdb_sd_configs:
+          - url: https://puppetdb.ls.lsst.org:8443
+            basic_auth:
+              username: svc_prometheus
+              password_file: /etc/prometheus/secrets/puppetdb/password
+            query: resources { type = "Class" and title = "Prometheus::Node_exporter" }
+            refresh_interval: 30s
+            follow_redirects: true
+            include_parameters: true
+            enable_http2: true
+        relabel_configs:
+          - source_labels: [__meta_puppetdb_certname]
+            target_label: __param_target
+          - source_labels: [__param_target]
+            target_label: instance
+          - target_label: __address__
+            replacement: prometheus-blackbox-exporter.blackbox-exporter:9115
+      - job_name: node-exporter-ls
+        puppetdb_sd_configs:
+          - url: https://puppetdb.ls.lsst.org:8443
+            basic_auth:
+              username: svc_prometheus
+              password_file: /etc/prometheus/secrets/puppetdb/password
+            query: |
+              resources {
+                type = "Class" and title = "Profile::Core::Node_info" and
+                certname in resources[certname] {
+                  type = "Class" and title = "Prometheus::Node_exporter"
+                }
+              }
+            refresh_interval: 30s
+            follow_redirects: true
+            include_parameters: true
+            enable_http2: true
+            port: 9100
+        relabel_configs:
+          - source_labels: [__meta_puppetdb_certname]
+            target_label: instance
+          - source_labels: [__meta_puppetdb_environment]
+            target_label: environment
+          - source_labels: [__meta_puppetdb_parameter_site]
+            target_label: site
+          - source_labels: [__meta_puppetdb_parameter_role]
+            target_label: role
+          - source_labels: [__meta_puppetdb_parameter_cluster]
+            target_label: cluster
+      - job_name: snmp-network
+        metrics_path: /snmp
+        params:
+          module: [if_mib]
+          auth: [rubin_v2]
+        file_sd_configs:
+          - files:
+              - /etc/prometheus/configmaps/sd-snmp-network/snmp-network.json
+        relabel_configs:
+          - source_labels: [__address__]
+            target_label: __param_target
+          - source_labels: [__meta_hostname]
+            target_label: instance
+          - source_labels: [__meta_network_function]
+            target_label: network_function
+          - target_label: __address__
+            replacement: prometheus-snmp-exporter.snmp-exporter:9116
+      - job_name: blackbox-network
+        metrics_path: /probe
+        params:
+          module: [icmp]
+        file_sd_configs:
+          - files:
+              - /etc/prometheus/configmaps/sd-snmp-network/snmp-network.json
+              - /etc/prometheus/configmaps/sd-snmp-power/snmp-power.json
+              - /etc/prometheus/configmaps/sd-snmp-raritan-pdu/snmp-raritan-pdu.json
+        relabel_configs:
+          - source_labels: [__address__]
+            target_label: __param_target
+          - source_labels: [__meta_hostname]
+            target_label: instance
+          - source_labels: [__meta_network_function]
+            target_label: network_function
+          - target_label: __address__
+            replacement: prometheus-blackbox-exporter.blackbox-exporter:9115
+      - job_name: snmp-power
+        metrics_path: /snmp
+        file_sd_configs:
+          - files:
+              - /etc/prometheus/configmaps/sd-snmp-power/snmp-power.json
+        relabel_configs:
+          - source_labels: [__address__]
+            target_label: __param_target
+          - source_labels: [__meta_hostname]
+            target_label: instance
+          - source_labels: [__meta_auth]
+            target_label: __param_auth
+          - source_labels: [__meta_module]
+            target_label: __param_module
+          - target_label: __address__
+            replacement: prometheus-snmp-exporter.snmp-exporter:9116
+      - job_name: snmp-raritan-pdu
+        metrics_path: /snmp
+        params:
+          module: [raritan]
+          auth: [rubin_v2]
+        file_sd_configs:
+          - files:
+              - /etc/prometheus/configmaps/sd-snmp-raritan-pdu/snmp-raritan-pdu.json
+        relabel_configs:
+          - source_labels: [__address__]
+            target_label: __param_target
+          - source_labels: [__meta_hostname]
+            target_label: instance
+
+grafana:
+  grafana.ini:
+    auth.generic_oauth:
+      api_url: https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/userinfo
+      auth_url: https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/auth
+      name: keycloak.ls.lsst.org
+      token_url: https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/token

fleet/lib/mimir-pre/fleet.yaml

@@ -49,3 +49,13 @@ targetCustomizations:
     yaml:
       overlays:
         - antu
+  - name: kueyen
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - kueyen
+    yaml:
+      overlays:
+        - kueyen

fleet/lib/mimir-pre/overlays/kueyen/externalsecret-mimir-s3.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: mimir-s3
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword
+  data:
+    - secretKey: AWS_ACCESS_KEY_ID
+      remoteRef:
+        key: &item mimir
+        property: AWS_ACCESS_KEY_ID
+    - secretKey: AWS_SECRET_ACCESS_KEY
+      remoteRef:
+        key: *item
+        property: AWS_SECRET_ACCESS_KEY

fleet/lib/mimir/fleet.yaml

@@ -32,6 +32,17 @@ targetCustomizations:
     helm:
       valuesFiles:
         - overlays/ayekan/values.yaml
+  - name: kueyen
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - kueyen
+    helm:
+      valuesFiles:
+        - overlays/kueyen/values.yaml
+        - overlays/rke2/values.yaml
   - name: rke2
     clusterSelector:
       matchExpressions:

fleet/lib/mimir/overlays/kueyen/values.yaml

@@ -0,0 +1,10 @@
+---
+mimir:
+  structuredConfig:
+    common:
+      storage:
+        s3:
+          region: o11y
+          endpoint: s3.o11y.kueyen.dev.lsst.org
+    limits:
+      compactor_blocks_retention_period: 90d

fleet/lib/rook-ceph-cluster/overlays/kueyen/values.yaml

@@ -6,27 +6,37 @@ cephClusterSpec:
     useAllNodes: false
     useAllDevices: false
     config:
-      osdsPerDevice: "4"
+      osdsPerDevice: "1"
     nodes:
       - name: kueyen01
         devices:
-          - name: /dev/nvme0n1
+          - name: /dev/disk/by-id/scsi-35000c500a15cf937
       - name: kueyen02
         devices:
-          - name: /dev/nvme0n1
+          - name: /dev/disk/by-id/scsi-35000c500a15847b7
       - name: kueyen03
         devices:
-          - name: /dev/nvme0n1
+          - name: /dev/disk/by-id/scsi-35000c500a15ca7af
+      - name: kueyen04
+        devices:
+          - name: /dev/disk/by-id/scsi-35000c500a15ce4d3
+          - name: /dev/disk/by-id/scsi-35000c500a1588b9b
+      - name: kueyen05
+        devices:
+          - name: /dev/disk/by-id/scsi-35000c500a157854b
+      - name: kueyen06
+        devices:
+          - name: /dev/disk/by-id/scsi-35000c500a1584f37
 
 cephBlockPools:
   - name: replicapool
     spec:
       failureDomain: host
       replicated:
-        size: 3
+        size: 2
         requireSafeReplicaSize: true
       quotas:
-        maxSize: 1.5Ti
+        maxSize: 2Ti
       enableRBDStats: true
     storageClass:
       name: rook-ceph-block

fleet/lib/rook-ceph-conf/charts/kueyen/templates/cephobjectstore-lfa.yaml renamed to fleet/lib/rook-ceph-conf/charts/kueyen/templates/cephobjectstore-o11y.yaml

@@ -2,7 +2,7 @@
 apiVersion: ceph.rook.io/v1
 kind: CephObjectStore
 metadata:
-  name: lfa
+  name: o11y
   namespace: rook-ceph
 spec:
   metadataPool:
@@ -17,36 +17,43 @@ spec:
       dataChunks: 2
       codingChunks: 1
     quotas:
-      maxSize: 1Ti
+      maxSize: 40Ti
   preservePoolsOnDelete: false
   gateway:
     sslCertificateRef:
     port: 80
     # securePort: 443
     instances: 3
+    resources:
+      limits:
+        cpu: "4"
+        memory: 4Gi
+      requests:
+        cpu: "4"
+        memory: 4Gi
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: rook-ceph-rgw-ingress
+  name: rook-ceph-rgw-ingress-o11y
   namespace: rook-ceph
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-staging
+    cert-manager.io/cluster-issuer: letsencrypt
     nginx.ingress.kubernetes.io/proxy-body-size: 1024m
 spec:
   ingressClassName: nginx
   tls:
   - hosts:
-    - s3.kueyen.dev.lsst.org
-    secretName: rook-ceph-rgw-ingress-tls
+    - &host s3.o11y.kueyen.dev.lsst.org
+    secretName: rook-ceph-rgw-ingress-tls-o11y
   rules:
-  - host: s3.kueyen.dev.lsst.org
+  - host: *host
     http:
       paths:
       - path: /
         pathType: Prefix
         backend:
           service:
-            name: rook-ceph-rgw-lfa
+            name: rook-ceph-rgw-o11y
             port:
               number: 80