(dev) Add fluentbit to dev site.

Author: gseriche

fleet/lib/fluent-bit-kube/fleet.yaml

@@ -17,3 +17,34 @@ dependsOn:
   - selector:
       matchLabels:
         bundle: fluent-bit-kube-pre
+targetCustomizations:
+  - name: kueyen
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - kueyen
+    helm:
+      valuesFiles:
+        - overlays/kueyen/values.yaml
+  - name: ayekan
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - ayekan
+    helm:
+      valuesFiles:
+        - overlays/ayekan/values.yaml
+  - name: ruka
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - ruka
+    helm:
+      valuesFiles:
+        - overlays/ruka/values.yaml

fleet/lib/fluent-bit-kube/overlays/ayekan/values.yaml

@@ -0,0 +1,14 @@
+config:
+    outputs: |
+        [Output]
+            Name              loki
+            Match_Regex       (?:kube|service|audit|syslog)\.(.*)
+            Host              loki.kueyen.dev.lsst.org
+            Port              443
+            TLS               On
+            TLS.Verify        Off
+            Labels            job=fluentbit,namespace=$kubernetes_namespace_name
+            Label_keys        $prom_cluster,$log_type
+            Line_Format        json
+            Auto_Kubernetes_Labels Off
+            Remove_keys      stream

fleet/lib/fluent-bit-kube/overlays/kueyen/values.yaml

@@ -0,0 +1,14 @@
+config:
+    outputs: |
+        [Output]
+            Name              loki
+            Match_Regex       (?:kube|service|audit|syslog)\.(.*)
+            Host              loki.kueyen.dev.lsst.org
+            Port              443
+            TLS               On
+            TLS.Verify        Off
+            Labels            job=fluentbit,namespace=$kubernetes_namespace_name
+            Label_keys        $prom_cluster,$log_type
+            Line_Format        json
+            Auto_Kubernetes_Labels Off
+            Remove_keys      stream

fleet/lib/fluent-bit-kube/overlays/ruka/values.yaml

@@ -0,0 +1,14 @@
+config:
+    outputs: |
+        [Output]
+            Name              loki
+            Match_Regex       (?:kube|service|audit|syslog)\.(.*)
+            Host              loki.kueyen.dev.lsst.org
+            Port              443
+            TLS               On
+            TLS.Verify        Off
+            Labels            job=fluentbit,namespace=$kubernetes_namespace_name
+            Label_keys        $prom_cluster,$log_type
+            Line_Format        json
+            Auto_Kubernetes_Labels Off
+            Remove_keys      stream

fleet/lib/fluent-bit-kube/values.yaml

@@ -13,6 +13,7 @@ serviceMonitor:
   enabled: true
   selector:
     lsst.io/monitor: "true"
+    prometheus.io/cluster: ${ get .ClusterLabels "management.cattle.io/cluster-display-name" }
   #   namespace: monitoring
   #   interval: 10s
   #   scrapeTimeout: 10s
@@ -227,6 +228,8 @@ config:
         HTTP_Listen 0.0.0.0
         HTTP_Port {{ .Values.metricsPort }}
         Health_Check On
+        HTTP_Max_Connections 200
+        HTTP_Buffer_Size 128k
 
   ## https://docs.fluentbit.io/manual/pipeline/inputs
   inputs: |
@@ -243,83 +246,100 @@ config:
         DB.Sync           Normal
 
     [INPUT]
-        Name              systemd
-        Tag               service.*
-        Path              /var/log/journal
-        DB                /fluent-bit/db/systemd.db
-        DB.Sync           Normal
-        Systemd_Filter    _SYSTEMD_UNIT=docker.service
-        Systemd_Filter    _SYSTEMD_UNIT=kubelet.service
-        Read_From_Tail    On
+        Name              tail
+        Tag               service.kubelet
+        Path              /var/lib/rancher/rke2/agent/logs/kubelet.log
+        DB                /fluent-bit/db/kubelet.db
+        Read_from_Head    false
 
   ## https://docs.fluentbit.io/manual/pipeline/filters
   filters: |
     [Filter]
-        Name                kubernetes
-        Match               kube.*
-        Kube_URL            https://kubernetes.default.svc:443
-        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
-        Merge_Log           On
-        Keep_Log            Off
-        K8S-Logging.Parser  On
+        Name kubernetes
+        Match kube.*
+        Kube_URL https://kubernetes.default.svc:443
+        Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
         K8S-Logging.Exclude On
-        Labels              true
-        Annotations         false
+        Labels true
+        Annotations false
+
     [Filter]
-        Name         nest
-        Match        kube.*
-        Operation    lift
+        Name nest
+        Match kube.*
+        Operation lift
         Nested_under kubernetes
-        Add_prefix   kubernetes_
+        Add_prefix kubernetes_
+
     [Filter]
-        Name   modify
-        Match  kube.*
-        Remove stream
+        Name modify
+        Match *
+        Add prom_cluster ${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }
+        Remove_wildcard kubernetes_labels_app*
+        Remove_wildcard kubernetes_labels_ceph*
+        Remove_wildcard kubernetes_labels_mgr*
+        Remove_wildcard kubernetes_labels_mon*
+        Remove_wildcard kubernetes_labels_rook*
+        Remove_wildcard kubernetes_labels_pod_template_hash
+        Remove_wildcard kubernetes_labels_controller_revision_hash
+        Remove pod_template_hash
         Remove kubernetes_pod_id
         Remove kubernetes_host
         Remove kubernetes_container_hash
+        Remove docker_id
+        Remove container_image
+        Remove _p
+        Remove pod_ip
+        Remove app_kubernetes_io_created_by
+        Remove app_kubernetes_io_managed_by
+        Remove app_kubernetes_io_instance
+        Remove app_kubernetes_io_name
+        Remove app_kubernetes_io_part_of
+        Remove controller_revision_hash
+        Remove helm_sh_chart
+        Remove service_name
+        Remove instance
+        Remove mgr
+        Remove mgr_role
+        Remove mon
+        Remove mon_cluster
+        Remove mon_daemon
+        Remove rook_io_operator_namespace
+
+    [Filter]
+        Name modify
+        Match kube.*
+        Add log_type kube_container
+
     [Filter]
-        Name          nest
-        Match         kube.*
-        Operation     nest
-        Wildcard      kubernetes_*
-        Nest_under    kubernetes
+        Name modify
+        Match service.kubelet
+        Add log_type kubelet_log
+
+    [Filter]
+        Name nest
+        Match kube.*
+        Operation nest
+        Wildcard kubernetes_*
+        Nest_under kubernetes
         Remove_prefix kubernetes_
+
     [Filter]
-        Name          lua
-        Match         kube.*
-        script        /fluent-bit/scripts/containerd.lua
-        call          containerd
+        Name lua
+        Match kube.*
+        script /fluent-bit/scripts/containerd.lua
+        call containerd
         time_as_table true
+
     [Filter]
-        Name          lua
-        Match         service.*
-        script        /fluent-bit/scripts/systemd.lua
-        call          add_time
+        Name lua
+        Match service.*
+        script /fluent-bit/scripts/systemd.lua
+        call add_time
         time_as_table true
-    [FILTER]
-        name  modify
-        match *
-        add   prom_cluster ${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }
-
-  ## https://docs.fluentbit.io/manual/pipeline/outputs
-  outputs: |
-    [Output]
-        Name               opensearch
-        Buffer_Size        1M
-        Match_Regex        (?:kube|service)\.(.*)
-        Host               logging.logging
-        Port               9200
-        HTTP_User          ${`${OS_LOGGING_USERNAME}`}
-        HTTP_Passwd        ${`${OS_LOGGING_PASSWORD}`}
-        Index              logs-kube
-        Write_Operation    create
-        Replace_Dots       true
-        Trace_Error        true
-        Suppress_Type_Name true
-        tls                On
-        tls.verify         false
 
   ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers
   ## This configuration is deprecated, please use `extraFiles` instead.
@@ -328,15 +348,31 @@ config:
   ## https://docs.fluentbit.io/manual/pipeline/parsers
   customParsers: |
     [PARSER]
-        Name docker_no_time
-        Format json
-        Time_Keep Off
-        Time_Key time
-        Time_Format %Y-%m-%dT%H:%M:%S.%L
+      Name        docker_no_time
+      Format      json
+      Time_Keep   Off
+      Time_Key    time
+      Time_Format %Y-%m-%dT%H:%M:%S.%L
+
+    [PARSER]
+        Name        auditd_line
+        Format      regex
+        Regex       ^node=(?<node>\S+)\s+type=(?<type>\S+)\s+msg=audit\((?<audit_time>[^)]+)\):(?<message>.*)
+        Time_Key    audit_time
+        Time_Format %s.%L
+
+    [PARSER]
+        Name        syslog_line
+        Format      regex
+        Regex       ^(?<timestamp>\d{4}-\d{2}-\d{2}T[^\s]+)\s+(?<host>\S+)\s+(?<program>[^\[]+)\[\d+\]:\s+\<(?<severity>\w+)\>\s+(?<message>.*)
+        Time_Key    timestamp
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
 
   # This allows adding more files with arbitrary filenames to /fluent-bit/etc/conf by providing key/value pairs.
   # The key becomes the filename, the value becomes the file content.
   extraFiles: {}
+
+  outputs: ""
 #     upstream.conf: |
 #       [UPSTREAM]
 #           upstream1