Skip to content

Commit edd890d

Browse files
committed
(dev) Add fluentbit to dev site.
1 parent 83ae2e0 commit edd890d

File tree

8 files changed

+125
-109
lines changed

8 files changed

+125
-109
lines changed

fleet/lib/fluent-bit-kube-pre/fleet.yaml

Lines changed: 0 additions & 9 deletions
This file was deleted.

fleet/lib/fluent-bit-kube-pre/manifests/externalsecret-fluentbit-kube-cred.yaml

Lines changed: 0 additions & 20 deletions
This file was deleted.

fleet/lib/fluent-bit-kube/fleet.yaml

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,11 @@ helm:
1313
waitForJobs: true
1414
valuesFiles:
1515
- values.yaml
16-
dependsOn:
17-
- selector:
16+
targetCustomizations:
17+
- name: dev
18+
clusterSelector:
1819
matchLabels:
19-
bundle: fluent-bit-kube-pre
20+
site: dev
21+
helm:
22+
valuesFiles:
23+
- overlays/dev/values.yaml
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
config:
2+
outputs: |
3+
[Output]
4+
Name loki
5+
Match_Regex (?:kube|service)\.(.*)
6+
Host loki.kueyen.dev.lsst.org
7+
Port 443
8+
TLS On
9+
TLS.Verify Off
10+
Labels job=fluentbit,namespace=$kubernetes_namespace_name
11+
Label_keys $prom_cluster,$log_type
12+
Line_Format json
13+
Auto_Kubernetes_Labels Off
14+
Remove_keys stream

fleet/lib/fluent-bit-kube/values.yaml

Lines changed: 101 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ serviceMonitor:
1313
enabled: true
1414
selector:
1515
lsst.io/monitor: "true"
16+
prometheus.io/cluster: ${ get .ClusterLabels "management.cattle.io/cluster-display-name" }
1617
# namespace: monitoring
1718
# interval: 10s
1819
# scrapeTimeout: 10s
@@ -108,18 +109,6 @@ terminationGracePeriodSeconds:
108109

109110
priorityClassName: ""
110111

111-
env:
112-
- name: OS_LOGGING_USERNAME
113-
valueFrom:
114-
secretKeyRef:
115-
name: fluentbit-kube-credentials
116-
key: username
117-
- name: OS_LOGGING_PASSWORD
118-
valueFrom:
119-
secretKeyRef:
120-
name: fluentbit-kube-credentials
121-
key: password
122-
123112
# The envWithTpl array below has the same usage as "env", but is using the tpl function to support templatable string.
124113
# This can be useful when you want to pass dynamic values to the Chart using the helm argument "--set <variable>=<value>"
125114
# https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function
@@ -227,6 +216,8 @@ config:
227216
HTTP_Listen 0.0.0.0
228217
HTTP_Port {{ .Values.metricsPort }}
229218
Health_Check On
219+
HTTP_Max_Connections 200
220+
HTTP_Buffer_Size 128k
230221
231222
## https://docs.fluentbit.io/manual/pipeline/inputs
232223
inputs: |
@@ -243,83 +234,100 @@ config:
243234
DB.Sync Normal
244235
245236
[INPUT]
246-
Name systemd
247-
Tag service.*
248-
Path /var/log/journal
249-
DB /fluent-bit/db/systemd.db
250-
DB.Sync Normal
251-
Systemd_Filter _SYSTEMD_UNIT=docker.service
252-
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
253-
Read_From_Tail On
237+
Name tail
238+
Tag service.kubelet
239+
Path /var/lib/rancher/rke2/agent/logs/kubelet.log
240+
DB /fluent-bit/db/kubelet.db
241+
Read_from_Head false
254242
255243
## https://docs.fluentbit.io/manual/pipeline/filters
256244
filters: |
257245
[Filter]
258-
Name kubernetes
259-
Match kube.*
260-
Kube_URL https://kubernetes.default.svc:443
261-
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
262-
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
263-
Merge_Log On
264-
Keep_Log Off
265-
K8S-Logging.Parser On
246+
Name kubernetes
247+
Match kube.*
248+
Kube_URL https://kubernetes.default.svc:443
249+
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
250+
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
251+
Merge_Log On
252+
Keep_Log Off
253+
K8S-Logging.Parser On
266254
K8S-Logging.Exclude On
267-
Labels true
268-
Annotations false
255+
Labels true
256+
Annotations false
257+
269258
[Filter]
270-
Name nest
271-
Match kube.*
272-
Operation lift
259+
Name nest
260+
Match kube.*
261+
Operation lift
273262
Nested_under kubernetes
274-
Add_prefix kubernetes_
263+
Add_prefix kubernetes_
264+
275265
[Filter]
276-
Name modify
277-
Match kube.*
278-
Remove stream
266+
Name modify
267+
Match *
268+
Add prom_cluster ${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }
269+
Remove_wildcard kubernetes_labels_app*
270+
Remove_wildcard kubernetes_labels_ceph*
271+
Remove_wildcard kubernetes_labels_mgr*
272+
Remove_wildcard kubernetes_labels_mon*
273+
Remove_wildcard kubernetes_labels_rook*
274+
Remove_wildcard kubernetes_labels_pod_template_hash
275+
Remove_wildcard kubernetes_labels_controller_revision_hash
276+
Remove pod_template_hash
279277
Remove kubernetes_pod_id
280278
Remove kubernetes_host
281279
Remove kubernetes_container_hash
280+
Remove docker_id
281+
Remove container_image
282+
Remove _p
283+
Remove pod_ip
284+
Remove app_kubernetes_io_created_by
285+
Remove app_kubernetes_io_managed_by
286+
Remove app_kubernetes_io_instance
287+
Remove app_kubernetes_io_name
288+
Remove app_kubernetes_io_part_of
289+
Remove controller_revision_hash
290+
Remove helm_sh_chart
291+
Remove service_name
292+
Remove instance
293+
Remove mgr
294+
Remove mgr_role
295+
Remove mon
296+
Remove mon_cluster
297+
Remove mon_daemon
298+
Remove rook_io_operator_namespace
299+
300+
[Filter]
301+
Name modify
302+
Match kube.*
303+
Add log_type kubernetes
304+
282305
[Filter]
283-
Name nest
284-
Match kube.*
285-
Operation nest
286-
Wildcard kubernetes_*
287-
Nest_under kubernetes
306+
Name modify
307+
Match service.kubelet
308+
Add log_type kubelet_log
309+
310+
[Filter]
311+
Name nest
312+
Match kube.*
313+
Operation nest
314+
Wildcard kubernetes_*
315+
Nest_under kubernetes
288316
Remove_prefix kubernetes_
317+
289318
[Filter]
290-
Name lua
291-
Match kube.*
292-
script /fluent-bit/scripts/containerd.lua
293-
call containerd
319+
Name lua
320+
Match kube.*
321+
script /fluent-bit/scripts/containerd.lua
322+
call containerd
294323
time_as_table true
324+
295325
[Filter]
296-
Name lua
297-
Match service.*
298-
script /fluent-bit/scripts/systemd.lua
299-
call add_time
326+
Name lua
327+
Match service.*
328+
script /fluent-bit/scripts/systemd.lua
329+
call add_time
300330
time_as_table true
301-
[FILTER]
302-
name modify
303-
match *
304-
add prom_cluster ${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }
305-
306-
## https://docs.fluentbit.io/manual/pipeline/outputs
307-
outputs: |
308-
[Output]
309-
Name opensearch
310-
Buffer_Size 1M
311-
Match_Regex (?:kube|service)\.(.*)
312-
Host logging.logging
313-
Port 9200
314-
HTTP_User ${`${OS_LOGGING_USERNAME}`}
315-
HTTP_Passwd ${`${OS_LOGGING_PASSWORD}`}
316-
Index logs-kube
317-
Write_Operation create
318-
Replace_Dots true
319-
Trace_Error true
320-
Suppress_Type_Name true
321-
tls On
322-
tls.verify false
323331
324332
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers
325333
## This configuration is deprecated, please use `extraFiles` instead.
@@ -328,15 +336,31 @@ config:
328336
## https://docs.fluentbit.io/manual/pipeline/parsers
329337
customParsers: |
330338
[PARSER]
331-
Name docker_no_time
332-
Format json
333-
Time_Keep Off
334-
Time_Key time
335-
Time_Format %Y-%m-%dT%H:%M:%S.%L
339+
Name docker_no_time
340+
Format json
341+
Time_Keep Off
342+
Time_Key time
343+
Time_Format %Y-%m-%dT%H:%M:%S.%L
344+
345+
[PARSER]
346+
Name auditd_line
347+
Format regex
348+
Regex ^node=(?<node>\S+)\s+type=(?<type>\S+)\s+msg=audit\((?<audit_time>[^)]+)\):(?<message>.*)
349+
Time_Key audit_time
350+
Time_Format %s.%L
351+
352+
[PARSER]
353+
Name syslog_line
354+
Format regex
355+
Regex ^(?<timestamp>\d{4}-\d{2}-\d{2}T[^\s]+)\s+(?<host>\S+)\s+(?<program>[^\[]+)\[\d+\]:\s+\<(?<severity>\w+)\>\s+(?<message>.*)
356+
Time_Key timestamp
357+
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
336358
337359
# This allows adding more files with arbitrary filenames to /fluent-bit/etc/conf by providing key/value pairs.
338360
# The key becomes the filename, the value becomes the file content.
339361
extraFiles: {}
362+
363+
outputs: ""
340364
# upstream.conf: |
341365
# [UPSTREAM]
342366
# upstream1

fleet/s/dev/c/ayekan/fluent-bit-kube

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../../lib/fluent-bit-kube

fleet/s/dev/c/kueyen/fluent-bit-kube

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../../lib/fluent-bit-kube

fleet/s/dev/c/ruka/fluent-bit-kube

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../../lib/fluent-bit-kube

0 commit comments

Comments
 (0)