From 11e125c6f66701f82432ecafb9c38461780b748c Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sat, 20 Sep 2025 20:23:52 +0800 Subject: [PATCH 01/40] =?UTF-8?q?=E5=8D=95=E6=9C=BA=E7=89=88=E6=9C=ACkafka?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0SASL=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 14 ++ charts/kafka/templates/broker/cm-jaas.yaml | 23 +++ .../templates/broker/job-init-scram.yaml | 51 +++++ .../kafka/templates/broker/secret-sasl.yaml | 15 ++ .../kafka/templates/broker/statefulset.yaml | 10 + charts/kafka/values.yaml | 16 ++ docs/sasl-scram.md | 192 ++++++++++++++++++ examples/kafka-client-sasl.yaml | 56 +++++ examples/values-sasl-scram.yml | 52 +++++ 9 files changed, 429 insertions(+) create mode 100644 charts/kafka/templates/broker/cm-jaas.yaml create mode 100644 charts/kafka/templates/broker/job-init-scram.yaml create mode 100644 charts/kafka/templates/broker/secret-sasl.yaml create mode 100644 docs/sasl-scram.md create mode 100644 examples/kafka-client-sasl.yaml create mode 100644 examples/values-sasl-scram.yml diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 77a00b1..2a921f8 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -136,7 +136,11 @@ broker env - name: KAFKA_CFG_ADVERTISED_LISTENERS value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + {{- if .Values.broker.auth.enabled }} + value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + {{- else }} value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + {{- end }} - name: KAFKA_CFG_INTER_BROKER_LISTENER_NAME value: BROKER - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES @@ -165,6 +169,16 @@ broker env - name: KAFKA_NODE_ID_OFFSET value: "1000" {{- end }} +{{- if .Values.broker.auth.enabled }} +- name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server_jaas.conf" +- name: KAFKA_CFG_SASL_ENABLED_MECHANISMS + value: {{ .Values.broker.auth.mechanism | quote }} +- name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL + value: {{ .Values.broker.auth.mechanism | quote }} +- name: KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL + value: "SASL_PLAINTEXT" +{{- end }} {{- if .Values.broker.external.enabled -}} {{- include "broker.externalEnv" $ | nindent 0 }} {{- end }} diff --git a/charts/kafka/templates/broker/cm-jaas.yaml b/charts/kafka/templates/broker/cm-jaas.yaml new file mode 100644 index 0000000..15fb2d0 --- /dev/null +++ b/charts/kafka/templates/broker/cm-jaas.yaml @@ -0,0 +1,23 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kafka.broker.fullname" . }}-jaas + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: broker +data: + kafka_server_jaas.conf: | + KafkaServer { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.broker.auth.users 0).username }}" + password="{{ (index .Values.broker.auth.users 0).password }}"; + }; + + Client { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.broker.auth.users 0).username }}" + password="{{ (index .Values.broker.auth.users 0).password }}"; + }; +{{- end }} \ No newline at end of file diff --git a/charts/kafka/templates/broker/job-init-scram.yaml b/charts/kafka/templates/broker/job-init-scram.yaml new file mode 100644 index 0000000..4343909 --- /dev/null +++ b/charts/kafka/templates/broker/job-init-scram.yaml @@ -0,0 +1,51 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "kafka.broker.fullname" . }}-init-scram + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: broker-init + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + {{- include "kafka.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: broker-init + spec: + restartPolicy: OnFailure + containers: + - name: init-scram-users + image: {{ include "kafka.kafkaImage" $ | quote }} + {{- with $.Values.image.pullPolicy }} + imagePullPolicy: {{ . | quote }} + {{- end }} + command: + - /bin/bash + - -c + - | + echo "Waiting for Kafka to be ready..." + while ! bin/kafka-broker-api-versions.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.containerPort }} >/dev/null 2>&1; do + echo "Kafka not ready yet, waiting..." + sleep 10 + done + + echo "Kafka is ready, creating SCRAM users..." + + {{- range .Values.broker.auth.users }} + # Create SCRAM user: {{ .username }} + bin/kafka-configs.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ $.Values.clusterDomain }}:{{ $.Values.broker.containerPort }} \ + --alter --add-config 'SCRAM-SHA-256=[password={{ .password }}]' \ + --entity-type users --entity-name {{ .username }} || true + {{- end }} + + echo "SCRAM users created successfully" + env: + - name: KAFKA_HEAP_OPTS + value: "-Xms256m -Xmx256m" +{{- end }} \ No newline at end of file diff --git a/charts/kafka/templates/broker/secret-sasl.yaml b/charts/kafka/templates/broker/secret-sasl.yaml new file mode 100644 index 0000000..c5619eb --- /dev/null +++ b/charts/kafka/templates/broker/secret-sasl.yaml @@ -0,0 +1,15 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kafka.broker.fullname" . }}-sasl + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: broker +type: Opaque +data: + {{- range .Values.broker.auth.users }} + {{ .username }}: {{ .password | b64enc | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kafka/templates/broker/statefulset.yaml b/charts/kafka/templates/broker/statefulset.yaml index af3d54d..db6ee59 100644 --- a/charts/kafka/templates/broker/statefulset.yaml +++ b/charts/kafka/templates/broker/statefulset.yaml @@ -98,6 +98,10 @@ spec: - mountPath: /entrypoint.sh name: entrypoint-sh subPath: entrypoint.sh + {{- if $.Values.broker.auth.enabled }} + - mountPath: "/etc/kafka/jaas" + name: jaas-config + {{- end }} {{- if eq (include "broker.external.autoDiscovery.enabled" $) "true" }} - name: auto-discovery-shared mountPath: /init-shared @@ -118,6 +122,12 @@ spec: path: entrypoint.sh name: {{ include "kafka.entrypoint.configmapName" $ }} defaultMode: 0744 + {{- if $.Values.broker.auth.enabled }} + - name: jaas-config + configMap: + name: {{ include "kafka.broker.fullname" $ }}-jaas + defaultMode: 0644 + {{- end }} {{- if eq (include "broker.external.autoDiscovery.enabled" $) "true" }} - name: auto-discovery-shared emptyDir: {} diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 41bd973..52bedb2 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -57,12 +57,28 @@ broker: commonEnvs: [] extraEnvs: [] + ## SASL SCRAM 认证配置 + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + # SASL用户凭据将通过Secret管理 + users: + - username: "kafka" + password: "Kafka@2025" + ## broker.config - server.properties 配置 ## 某些关键配置会被环境变量覆盖，例如: node.id advertised.listeners controller.quorum.voters 等 config: num.partitions: "2" # default.replication.factor: "1" # min.insync.replicas: "1" + # SASL相关配置 + sasl.enabled.mechanisms: "SCRAM-SHA-256" + sasl.mechanism.inter.broker.protocol: "SCRAM-SHA-256" + security.inter.broker.protocol: "SASL_PLAINTEXT" + listener.security.protocol.map: "CONTROLLER:PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT" + listeners: "CONTROLLER://:19091,PLAINTEXT://:9092" + advertised.listeners: "PLAINTEXT://localhost:9092" ## broker.persistence persistence: diff --git a/docs/sasl-scram.md b/docs/sasl-scram.md new file mode 100644 index 0000000..1b92412 --- /dev/null +++ b/docs/sasl-scram.md @@ -0,0 +1,192 @@ +# Kafka SASL SCRAM 认证配置指南 + +本文档介绍如何在 Kafka Helm Chart 中配置 SASL SCRAM 认证。 + +## 概述 + +SASL SCRAM (Salted Challenge Response Authentication Mechanism) 是一种安全的认证机制，提供了比 PLAIN 认证更强的安全性。本配置支持 SCRAM-SHA-256 机制。 + +## 配置说明 + +### 1. 启用 SASL SCRAM 认证 + +在 `values.yaml` 中添加以下配置： + +```yaml +broker: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" +``` + +### 2. 自动配置的参数 + +启用 SASL SCRAM 认证后，以下参数会自动配置： + +- `sasl.enabled.mechanisms=SCRAM-SHA-256` +- `sasl.mechanism.inter.broker.protocol=SCRAM-SHA-256` +- `security.inter.broker.protocol=SASL_PLAINTEXT` +- `listener.security.protocol.map=CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT` + +## 部署步骤 + +### 1. 使用示例配置部署 + +```bash +# 使用预配置的 SASL SCRAM 配置文件 +helm install kafka-sasl ./charts/kafka -f examples/values-sasl-scram.yml + +# 或者自定义配置 +helm install kafka-sasl ./charts/kafka --set broker.auth.enabled=true \ + --set broker.auth.mechanism=SCRAM-SHA-256 \ + --set broker.auth.users[0].username=kafka \ + --set broker.auth.users[0].password=Kafka@2025 +``` + +### 2. 验证部署状态 + +```bash +# 检查 Pod 状态 +kubectl get pods -l app.kubernetes.io/name=kafka + +# 检查 SCRAM 用户初始化 Job +kubectl get jobs -l app.kubernetes.io/component=broker-init + +# 查看 Kafka 日志 +kubectl logs -l app.kubernetes.io/component=broker +``` + +## 测试连接 + +### 1. 部署测试客户端 + +```bash +# 部署测试客户端 Pod +kubectl apply -f examples/kafka-client-sasl.yaml +``` + +### 2. 测试生产者 + +```bash +# 进入客户端 Pod +kubectl exec -it kafka-client-sasl -- bash + +# 创建主题 +bin/kafka-topics.sh --bootstrap-server kafka-sasl:9092 \ + --command-config /etc/kafka/client/client.properties \ + --create --topic test-topic --partitions 2 --replication-factor 1 + +# 发送消息 +echo "Hello SASL SCRAM" | bin/kafka-console-producer.sh \ + --bootstrap-server kafka-sasl:9092 \ + --producer.config /etc/kafka/client/producer.properties \ + --topic test-topic +``` + +### 3. 测试消费者 + +```bash +# 消费消息 +bin/kafka-console-consumer.sh \ + --bootstrap-server kafka-sasl:9092 \ + --consumer.config /etc/kafka/client/consumer.properties \ + --topic test-topic --from-beginning +``` + +## 客户端配置示例 + +### Java 客户端配置 + +```properties +bootstrap.servers=kafka-sasl:9092 +security.protocol=SASL_PLAINTEXT +sasl.mechanism=SCRAM-SHA-256 +sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; +``` + +### Python 客户端配置 (kafka-python) + +```python +from kafka import KafkaProducer, KafkaConsumer + +producer = KafkaProducer( + bootstrap_servers=['kafka-sasl:9092'], + security_protocol='SASL_PLAINTEXT', + sasl_mechanism='SCRAM-SHA-256', + sasl_plain_username='kafka', + sasl_plain_password='Kafka@2025' +) + +consumer = KafkaConsumer( + 'test-topic', + bootstrap_servers=['kafka-sasl:9092'], + security_protocol='SASL_PLAINTEXT', + sasl_mechanism='SCRAM-SHA-256', + sasl_plain_username='kafka', + sasl_plain_password='Kafka@2025', + group_id='test-group' +) +``` + +## 安全注意事项 + +1. **密码管理**: 在生产环境中，建议使用 Kubernetes Secret 来管理密码，而不是直接在 values.yaml 中明文存储。 + +2. **网络安全**: 当前配置使用 SASL_PLAINTEXT，在生产环境中建议使用 SASL_SSL 来加密网络传输。 + +3. **用户权限**: 可以通过 Kafka ACL 来进一步限制用户权限。 + +## 故障排除 + +### 1. 认证失败 + +如果遇到认证失败，检查： +- SCRAM 用户是否正确创建 +- 客户端配置中的用户名和密码是否正确 +- JAAS 配置是否正确挂载 + +### 2. 连接超时 + +如果连接超时，检查： +- Kafka 服务是否正常运行 +- 网络连接是否正常 +- 端口配置是否正确 + +### 3. 查看详细日志 + +```bash +# 查看 Kafka broker 日志 +kubectl logs -l app.kubernetes.io/component=broker -f + +# 查看 SCRAM 初始化 Job 日志 +kubectl logs job/kafka-sasl-broker-init-scram +``` + +## 升级和维护 + +### 添加新用户 + +要添加新的 SCRAM 用户，可以： + +1. 更新 values.yaml 中的用户列表 +2. 执行 Helm 升级： + +```bash +helm upgrade kafka-sasl ./charts/kafka -f examples/values-sasl-scram.yml +``` + +### 修改密码 + +修改现有用户密码： + +1. 更新 values.yaml 中的密码 +2. 执行 Helm 升级 +3. SCRAM 初始化 Job 会自动更新用户凭据 + +## 参考资料 + +- [Apache Kafka SASL/SCRAM 文档](https://kafka.apache.org/documentation/#security_sasl_scram) +- [Kafka 安全配置最佳实践](https://kafka.apache.org/documentation/#security) \ No newline at end of file diff --git a/examples/kafka-client-sasl.yaml b/examples/kafka-client-sasl.yaml new file mode 100644 index 0000000..da56a07 --- /dev/null +++ b/examples/kafka-client-sasl.yaml @@ -0,0 +1,56 @@ +# Kafka 客户端 SASL SCRAM 认证示例 +# 用于测试 Kafka SASL SCRAM 认证连接 + +apiVersion: v1 +kind: Pod +metadata: + name: kafka-client-sasl + labels: + app: kafka-client-sasl +spec: + containers: + - name: kafka-client + image: kafkace/kafka:v3.7.1-63ba8d2 + command: + - sleep + - "3600" + env: + - name: KAFKA_HEAP_OPTS + value: "-Xms256m -Xmx256m" + volumeMounts: + - name: client-config + mountPath: /etc/kafka/client + volumes: + - name: client-config + configMap: + name: kafka-client-sasl-config + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-client-sasl-config +data: + client.properties: | + bootstrap.servers=kafka-sasl:9092 + security.protocol=SASL_PLAINTEXT + sasl.mechanism=SCRAM-SHA-256 + sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; + + producer.properties: | + bootstrap.servers=kafka-sasl:9092 + security.protocol=SASL_PLAINTEXT + sasl.mechanism=SCRAM-SHA-256 + sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; + key.serializer=org.apache.kafka.common.serialization.StringSerializer + value.serializer=org.apache.kafka.common.serialization.StringSerializer + + consumer.properties: | + bootstrap.servers=kafka-sasl:9092 + security.protocol=SASL_PLAINTEXT + sasl.mechanism=SCRAM-SHA-256 + sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; + key.deserializer=org.apache.kafka.common.serialization.StringDeserializer + value.deserializer=org.apache.kafka.common.serialization.StringDeserializer + group.id=test-consumer-group + auto.offset.reset=earliest \ No newline at end of file diff --git a/examples/values-sasl-scram.yml b/examples/values-sasl-scram.yml new file mode 100644 index 0000000..d7429dd --- /dev/null +++ b/examples/values-sasl-scram.yml @@ -0,0 +1,52 @@ +# Kafka SASL SCRAM 认证配置示例 +# 使用方法: helm install kafka ./charts/kafka -f examples/values-sasl-scram.yml + +# 基础配置 +nameOverride: "kafka-sasl" +fullnameOverride: "kafka-sasl" + +# Kafka Broker 配置 +broker: + replicaCount: 1 + + # 启用 SASL SCRAM 认证 + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" + + # Kafka 配置 + config: + num.partitions: "2" + # SASL 相关配置会自动添加 + + # 资源配置 + resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + + # 持久化存储 + persistence: + enabled: true + size: 10Gi + + # 服务配置 + service: + type: ClusterIP + port: 9092 + +# 禁用其他组件（单机版本） +controller: + enabled: false + +exporter: + enabled: false + +ui: + enabled: false \ No newline at end of file From 014ca61eed891a26fe9ba537376382f6ce0a73ff Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sat, 20 Sep 2025 21:45:04 +0800 Subject: [PATCH 02/40] =?UTF-8?q?=E4=BF=AE=E5=A4=8DKafka=20SASL=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E5=86=B2=E7=AA=81=EF=BC=9A=E7=A7=BB=E9=99=A4=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E7=9A=84inter.broker=E5=8D=8F=E8=AE=AE=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 2 -- charts/kafka/values.yaml | 7 ------- 2 files changed, 9 deletions(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 2a921f8..12b629e 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -176,8 +176,6 @@ broker env value: {{ .Values.broker.auth.mechanism | quote }} - name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL value: {{ .Values.broker.auth.mechanism | quote }} -- name: KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL - value: "SASL_PLAINTEXT" {{- end }} {{- if .Values.broker.external.enabled -}} {{- include "broker.externalEnv" $ | nindent 0 }} diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 52bedb2..b13ce08 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -72,13 +72,6 @@ broker: num.partitions: "2" # default.replication.factor: "1" # min.insync.replicas: "1" - # SASL相关配置 - sasl.enabled.mechanisms: "SCRAM-SHA-256" - sasl.mechanism.inter.broker.protocol: "SCRAM-SHA-256" - security.inter.broker.protocol: "SASL_PLAINTEXT" - listener.security.protocol.map: "CONTROLLER:PLAINTEXT,PLAINTEXT:SASL_PLAINTEXT" - listeners: "CONTROLLER://:19091,PLAINTEXT://:9092" - advertised.listeners: "PLAINTEXT://localhost:9092" ## broker.persistence persistence: From 7f3731ac992a6e22b57254144fe765a3c557ab3c Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sat, 20 Sep 2025 22:14:24 +0800 Subject: [PATCH 03/40] =?UTF-8?q?=E4=BF=AE=E5=A4=8DSASL=E6=8F=A1=E6=89=8B?= =?UTF-8?q?=E9=94=99=E8=AF=AF=EF=BC=9A=E8=B0=83=E6=95=B4=E7=9B=91=E5=90=AC?= =?UTF-8?q?=E5=99=A8=E5=AE=89=E5=85=A8=E5=8D=8F=E8=AE=AE=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=92=8C=E8=BF=9E=E6=8E=A5=E7=AB=AF=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 2 +- charts/kafka/templates/broker/job-init-scram.yaml | 4 ++-- examples/kafka-client-sasl.yaml | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 12b629e..5b4e17f 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -137,7 +137,7 @@ broker env value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP {{- if .Values.broker.auth.enabled }} - value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:PLAINTEXT {{- else }} value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT {{- end }} diff --git a/charts/kafka/templates/broker/job-init-scram.yaml b/charts/kafka/templates/broker/job-init-scram.yaml index 4343909..a91ca68 100644 --- a/charts/kafka/templates/broker/job-init-scram.yaml +++ b/charts/kafka/templates/broker/job-init-scram.yaml @@ -30,7 +30,7 @@ spec: - -c - | echo "Waiting for Kafka to be ready..." - while ! bin/kafka-broker-api-versions.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.containerPort }} >/dev/null 2>&1; do + while ! bin/kafka-broker-api-versions.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.external.containerPort }} >/dev/null 2>&1; do echo "Kafka not ready yet, waiting..." sleep 10 done @@ -39,7 +39,7 @@ spec: {{- range .Values.broker.auth.users }} # Create SCRAM user: {{ .username }} - bin/kafka-configs.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ $.Values.clusterDomain }}:{{ $.Values.broker.containerPort }} \ + bin/kafka-configs.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ $.Values.clusterDomain }}:{{ $.Values.broker.external.containerPort }} \ --alter --add-config 'SCRAM-SHA-256=[password={{ .password }}]' \ --entity-type users --entity-name {{ .username }} || true {{- end }} diff --git a/examples/kafka-client-sasl.yaml b/examples/kafka-client-sasl.yaml index da56a07..9e2ffa6 100644 --- a/examples/kafka-client-sasl.yaml +++ b/examples/kafka-client-sasl.yaml @@ -32,13 +32,13 @@ metadata: name: kafka-client-sasl-config data: client.properties: | - bootstrap.servers=kafka-sasl:9092 + bootstrap.servers=kafka-sasl-broker:9092 security.protocol=SASL_PLAINTEXT sasl.mechanism=SCRAM-SHA-256 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; producer.properties: | - bootstrap.servers=kafka-sasl:9092 + bootstrap.servers=kafka-sasl-broker:9092 security.protocol=SASL_PLAINTEXT sasl.mechanism=SCRAM-SHA-256 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; @@ -46,7 +46,7 @@ data: value.serializer=org.apache.kafka.common.serialization.StringSerializer consumer.properties: | - bootstrap.servers=kafka-sasl:9092 + bootstrap.servers=kafka-sasl-broker:9092 security.protocol=SASL_PLAINTEXT sasl.mechanism=SCRAM-SHA-256 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; From 3b149f20e0dfcc90fa7eaf3fa20579ba3aa37023 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sat, 20 Sep 2025 22:15:16 +0800 Subject: [PATCH 04/40] =?UTF-8?q?=E6=9B=B4=E6=96=B0SASL=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E8=BF=9E=E6=8E=A5=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/sasl-scram.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sasl-scram.md b/docs/sasl-scram.md index 1b92412..69caee3 100644 --- a/docs/sasl-scram.md +++ b/docs/sasl-scram.md @@ -75,13 +75,13 @@ kubectl apply -f examples/kafka-client-sasl.yaml kubectl exec -it kafka-client-sasl -- bash # 创建主题 -bin/kafka-topics.sh --bootstrap-server kafka-sasl:9092 \ +bin/kafka-topics.sh --bootstrap-server kafka-sasl-broker:9092 \ --command-config /etc/kafka/client/client.properties \ --create --topic test-topic --partitions 2 --replication-factor 1 # 发送消息 echo "Hello SASL SCRAM" | bin/kafka-console-producer.sh \ - --bootstrap-server kafka-sasl:9092 \ + --bootstrap-server kafka-sasl-broker:9092 \ --producer.config /etc/kafka/client/producer.properties \ --topic test-topic ``` @@ -91,7 +91,7 @@ echo "Hello SASL SCRAM" | bin/kafka-console-producer.sh \ ```bash # 消费消息 bin/kafka-console-consumer.sh \ - --bootstrap-server kafka-sasl:9092 \ + --bootstrap-server kafka-sasl-broker:9092 \ --consumer.config /etc/kafka/client/consumer.properties \ --topic test-topic --from-beginning ``` From 1301a2313db4192708df3133b80f3e4133cb3a74 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sat, 20 Sep 2025 23:16:15 +0800 Subject: [PATCH 05/40] =?UTF-8?q?=E4=BF=AE=E5=A4=8DKafka=20PLAINTEXT=20?= =?UTF-8?q?=E7=AB=AF=E5=8F=A3=209094?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 25 +++++++++++++++---- .../templates/broker/job-init-scram.yaml | 4 +-- charts/kafka/templates/broker/svc.yaml | 4 +++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 5b4e17f..9d51e60 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -15,6 +15,9 @@ broker.containerPorts {{- end }} {{- end }} {{- end }} +- containerPort: 9094 + name: plaintext + protocol: TCP {{- if not .Values.controller.enabled }} - containerPort: {{ .Values.controller.containerPort }} name: controller @@ -96,11 +99,19 @@ broker.advertisedListeners.external {{- printf "EXTERNAL://%s:%d" $addr ($port | int) -}} {{- end -}} +{{/* +broker.advertisedListeners.plaintext +*/}} +{{- define "broker.advertisedListeners.plaintext" -}} +{{- $serviceAddr := (include "broker.headless.serviceAddr" .) -}} +{{- printf "PLAINTEXT://$(POD_NAME).%s:9094" $serviceAddr -}} +{{- end -}} + {{/* broker.config.advertised.listeners */}} {{- define "broker.config.advertised.listeners" -}} -{{- printf "%s,%s" (include "broker.advertisedListeners.internal" .) (include "broker.advertisedListeners.external" .) -}} +{{- printf "%s,%s,%s" (include "broker.advertisedListeners.internal" .) (include "broker.advertisedListeners.external" .) (include "broker.advertisedListeners.plaintext" .) -}} {{- end -}} {{/* @@ -129,20 +140,24 @@ broker env {{- end }} - name: KAFKA_CFG_LISTENERS {{- if not .Values.controller.enabled }} - value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }},CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" + value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }},CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }},PLAINTEXT://0.0.0.0:9094" {{- else }} - value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }}" + value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }},PLAINTEXT://0.0.0.0:9094" {{- end }} - name: KAFKA_CFG_ADVERTISED_LISTENERS value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP {{- if .Values.broker.auth.enabled }} - value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:PLAINTEXT + value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT {{- else }} - value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT {{- end }} - name: KAFKA_CFG_INTER_BROKER_LISTENER_NAME + {{- if .Values.broker.auth.enabled }} + value: PLAINTEXT + {{- else }} value: BROKER + {{- end }} - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS diff --git a/charts/kafka/templates/broker/job-init-scram.yaml b/charts/kafka/templates/broker/job-init-scram.yaml index a91ca68..566abaf 100644 --- a/charts/kafka/templates/broker/job-init-scram.yaml +++ b/charts/kafka/templates/broker/job-init-scram.yaml @@ -30,7 +30,7 @@ spec: - -c - | echo "Waiting for Kafka to be ready..." - while ! bin/kafka-broker-api-versions.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ .Values.clusterDomain }}:{{ .Values.broker.external.containerPort }} >/dev/null 2>&1; do + while ! bin/kafka-broker-api-versions.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ .Values.clusterDomain }}:9094 >/dev/null 2>&1; do echo "Kafka not ready yet, waiting..." sleep 10 done @@ -39,7 +39,7 @@ spec: {{- range .Values.broker.auth.users }} # Create SCRAM user: {{ .username }} - bin/kafka-configs.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ $.Values.clusterDomain }}:{{ $.Values.broker.external.containerPort }} \ + bin/kafka-configs.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ $.Values.clusterDomain }}:9094 \ --alter --add-config 'SCRAM-SHA-256=[password={{ .password }}]' \ --entity-type users --entity-name {{ .username }} || true {{- end }} diff --git a/charts/kafka/templates/broker/svc.yaml b/charts/kafka/templates/broker/svc.yaml index d33de7f..ad98992 100644 --- a/charts/kafka/templates/broker/svc.yaml +++ b/charts/kafka/templates/broker/svc.yaml @@ -33,6 +33,10 @@ spec: nodePort: {{ .nodePort }} {{- end }} {{- end }} + - name: plaintext + port: 9094 + targetPort: plaintext + protocol: TCP {{- end }} selector: {{- include "kafka.selectorLabels" $ | nindent 4 }} From a59eb8660df89192d46395bb001a74c00c57152b Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sat, 20 Sep 2025 23:39:01 +0800 Subject: [PATCH 06/40] . --- charts/kafka/templates/broker/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 9d51e60..93c5cc3 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -148,7 +148,7 @@ broker env value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP {{- if .Values.broker.auth.enabled }} - value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT + value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,PLAINTEXT:PLAINTEXT {{- else }} value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT {{- end }} From 8d89b4f900f1ffedd2fc8b5e9959ec2c37a340ab Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 00:04:14 +0800 Subject: [PATCH 07/40] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=BA=86=E5=81=A5?= =?UTF-8?q?=E5=BA=B7=E6=A3=80=E6=9F=A5=E4=B8=8E=E8=AE=A4=E8=AF=81=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=8D=E5=8C=B9=E9=85=8D=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kafka/templates/broker/statefulset.yaml b/charts/kafka/templates/broker/statefulset.yaml index db6ee59..f63caf2 100644 --- a/charts/kafka/templates/broker/statefulset.yaml +++ b/charts/kafka/templates/broker/statefulset.yaml @@ -74,7 +74,7 @@ spec: command: - sh - -c - - bin/kafka-broker-api-versions.sh --bootstrap-server=127.0.0.1:{{ $.Values.broker.containerPort }} + - bin/kafka-broker-api-versions.sh --bootstrap-server=127.0.0.1:9094 {{- with .readinessProbe }} {{- toYaml . | nindent 10 }} {{- end }} From 5da1db51b86ad9c62e74e6aa150322cd3b5ccbeb Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 01:32:16 +0800 Subject: [PATCH 08/40] =?UTF-8?q?=E8=A7=A3=E5=86=B3kafka-client-sasl-confi?= =?UTF-8?q?g?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/broker/cm-client-config.yaml | 13 +++++ .../kafka/templates/broker/statefulset.yaml | 8 +++ examples/kafka-client-sasl.yaml | 56 ------------------- 3 files changed, 21 insertions(+), 56 deletions(-) create mode 100644 charts/kafka/templates/broker/cm-client-config.yaml delete mode 100644 examples/kafka-client-sasl.yaml diff --git a/charts/kafka/templates/broker/cm-client-config.yaml b/charts/kafka/templates/broker/cm-client-config.yaml new file mode 100644 index 0000000..864d2f6 --- /dev/null +++ b/charts/kafka/templates/broker/cm-client-config.yaml @@ -0,0 +1,13 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-client-sasl-config + labels: + {{- include "kafka.labels" . | nindent 4 }} +data: + client.properties: | + security.protocol=SASL_PLAINTEXT + sasl.mechanism=SCRAM-SHA-256 + sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="{{ (index .Values.broker.auth.users 0).username }}" password="{{ (index .Values.broker.auth.users 0).password }}"; +{{- end }} \ No newline at end of file diff --git a/charts/kafka/templates/broker/statefulset.yaml b/charts/kafka/templates/broker/statefulset.yaml index f63caf2..bca0a8b 100644 --- a/charts/kafka/templates/broker/statefulset.yaml +++ b/charts/kafka/templates/broker/statefulset.yaml @@ -99,6 +99,10 @@ spec: name: entrypoint-sh subPath: entrypoint.sh {{- if $.Values.broker.auth.enabled }} + - mountPath: /opt/kafka/config + name: client-config + {{- end }} + {{- if $.Values.broker.auth.enabled }} - mountPath: "/etc/kafka/jaas" name: jaas-config {{- end }} @@ -127,6 +131,10 @@ spec: configMap: name: {{ include "kafka.broker.fullname" $ }}-jaas defaultMode: 0644 + - name: client-config + configMap: + name: kafka-client-sasl-config + defaultMode: 0644 {{- end }} {{- if eq (include "broker.external.autoDiscovery.enabled" $) "true" }} - name: auto-discovery-shared diff --git a/examples/kafka-client-sasl.yaml b/examples/kafka-client-sasl.yaml deleted file mode 100644 index 9e2ffa6..0000000 --- a/examples/kafka-client-sasl.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Kafka 客户端 SASL SCRAM 认证示例 -# 用于测试 Kafka SASL SCRAM 认证连接 - -apiVersion: v1 -kind: Pod -metadata: - name: kafka-client-sasl - labels: - app: kafka-client-sasl -spec: - containers: - - name: kafka-client - image: kafkace/kafka:v3.7.1-63ba8d2 - command: - - sleep - - "3600" - env: - - name: KAFKA_HEAP_OPTS - value: "-Xms256m -Xmx256m" - volumeMounts: - - name: client-config - mountPath: /etc/kafka/client - volumes: - - name: client-config - configMap: - name: kafka-client-sasl-config - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kafka-client-sasl-config -data: - client.properties: | - bootstrap.servers=kafka-sasl-broker:9092 - security.protocol=SASL_PLAINTEXT - sasl.mechanism=SCRAM-SHA-256 - sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; - - producer.properties: | - bootstrap.servers=kafka-sasl-broker:9092 - security.protocol=SASL_PLAINTEXT - sasl.mechanism=SCRAM-SHA-256 - sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; - key.serializer=org.apache.kafka.common.serialization.StringSerializer - value.serializer=org.apache.kafka.common.serialization.StringSerializer - - consumer.properties: | - bootstrap.servers=kafka-sasl-broker:9092 - security.protocol=SASL_PLAINTEXT - sasl.mechanism=SCRAM-SHA-256 - sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; - key.deserializer=org.apache.kafka.common.serialization.StringDeserializer - value.deserializer=org.apache.kafka.common.serialization.StringDeserializer - group.id=test-consumer-group - auto.offset.reset=earliest \ No newline at end of file From ca854f0514846f450661fbadb5b3738e5ac739b9 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 02:03:55 +0800 Subject: [PATCH 09/40] =?UTF-8?q?external=20nodePort=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 15 ++++++++++++++- charts/kafka/templates/broker/statefulset.yaml | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 93c5cc3..14b50c2 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -88,6 +88,7 @@ broker.advertisedListeners.external {{- with .Values.broker.external -}} {{- if eq .type "NodePort" -}} {{- $addr = "$(POD_HOST_IP)" -}} + {{- $port = "$(KAFKA_EXTERNAL_NODEPORT)" -}} {{- else if eq .type "HostPort" -}} {{- $addr = "$(POD_HOST_IP)" -}} {{- $port = .hostPort | default .containerPort -}} @@ -96,7 +97,7 @@ broker.advertisedListeners.external {{- $addr = printf "$(POD_NAME).%s.%s" (include "broker.externalDns.hostnamePrefix" $) (include "broker.externalDns.domain" $) -}} {{- end -}} {{- end -}} -{{- printf "EXTERNAL://%s:%d" $addr ($port | int) -}} +{{- printf "EXTERNAL://%s:%s" $addr $port -}} {{- end -}} {{/* @@ -130,6 +131,10 @@ broker env valueFrom: fieldRef: fieldPath: metadata.name +{{- if and .Values.broker.external.enabled (eq .Values.broker.external.type "NodePort") }} +- name: KAFKA_EXTERNAL_NODEPORT + value: "{{ include "broker.external.nodePort" . }}" +{{- end }} - name: KAFKA_HEAP_OPTS value: {{ .Values.broker.heapOpts | quote }} - name: KAFKA_CFG_PROCESS_ROLES @@ -227,6 +232,14 @@ broker.externalEnv {{- end }} {{- end }} +{{/* +broker.external.nodePort +*/}} +{{- define "broker.external.nodePort" -}} +{{- $fullNodePorts := include "broker.fullNodePorts" . -}} +{{- printf "$(echo '%s' | cut -d',' -f$(($(echo $POD_NAME | sed 's/.*-//')+1)))" $fullNodePorts -}} +{{- end -}} + {{/* broker.fullNodePorts */}} diff --git a/charts/kafka/templates/broker/statefulset.yaml b/charts/kafka/templates/broker/statefulset.yaml index bca0a8b..614af4a 100644 --- a/charts/kafka/templates/broker/statefulset.yaml +++ b/charts/kafka/templates/broker/statefulset.yaml @@ -99,8 +99,9 @@ spec: name: entrypoint-sh subPath: entrypoint.sh {{- if $.Values.broker.auth.enabled }} - - mountPath: /opt/kafka/config + - mountPath: /opt/kafka/config/client.properties name: client-config + subPath: client.properties {{- end }} {{- if $.Values.broker.auth.enabled }} - mountPath: "/etc/kafka/jaas" From ae63eb06a38b9d394a0c5a2e9685e29c3157060e Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 02:19:32 +0800 Subject: [PATCH 10/40] =?UTF-8?q?external=20nodePort=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 14b50c2..6912e23 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -86,9 +86,11 @@ broker.advertisedListeners.external {{- $addr := "$(POD_IP)" -}} {{- $port := .Values.broker.external.containerPort | int -}} {{- with .Values.broker.external -}} +{{- if .enabled -}} {{- if eq .type "NodePort" -}} {{- $addr = "$(POD_HOST_IP)" -}} - {{- $port = "$(KAFKA_EXTERNAL_NODEPORT)" -}} + {{- $fullNodePorts := include "broker.fullNodePorts" $ -}} + {{- $port = printf "$(echo '%s' | cut -d',' -f$(($(echo $POD_NAME | sed 's/.*-//')+1)))" $fullNodePorts -}} {{- else if eq .type "HostPort" -}} {{- $addr = "$(POD_HOST_IP)" -}} {{- $port = .hostPort | default .containerPort -}} @@ -97,6 +99,7 @@ broker.advertisedListeners.external {{- $addr = printf "$(POD_NAME).%s.%s" (include "broker.externalDns.hostnamePrefix" $) (include "broker.externalDns.domain" $) -}} {{- end -}} {{- end -}} +{{- end -}} {{- printf "EXTERNAL://%s:%s" $addr $port -}} {{- end -}} @@ -112,7 +115,11 @@ broker.advertisedListeners.plaintext broker.config.advertised.listeners */}} {{- define "broker.config.advertised.listeners" -}} +{{- if .Values.broker.external.enabled -}} {{- printf "%s,%s,%s" (include "broker.advertisedListeners.internal" .) (include "broker.advertisedListeners.external" .) (include "broker.advertisedListeners.plaintext" .) -}} +{{- else -}} +{{- printf "%s,%s" (include "broker.advertisedListeners.internal" .) (include "broker.advertisedListeners.plaintext" .) -}} +{{- end -}} {{- end -}} {{/* From 2b3f170fc75af9f796322ea5c09572643bc0328e Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 02:57:54 +0800 Subject: [PATCH 11/40] =?UTF-8?q?external=20nodePort=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 6912e23..d84201e 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -89,8 +89,7 @@ broker.advertisedListeners.external {{- if .enabled -}} {{- if eq .type "NodePort" -}} {{- $addr = "$(POD_HOST_IP)" -}} - {{- $fullNodePorts := include "broker.fullNodePorts" $ -}} - {{- $port = printf "$(echo '%s' | cut -d',' -f$(($(echo $POD_NAME | sed 's/.*-//')+1)))" $fullNodePorts -}} + {{- $port = "$(echo $KAFKA_EXTERNAL_NODEPORTS | cut -d',' -f$(($(echo $POD_NAME | sed 's/.*-//')+1)))" -}} {{- else if eq .type "HostPort" -}} {{- $addr = "$(POD_HOST_IP)" -}} {{- $port = .hostPort | default .containerPort -}} @@ -139,8 +138,8 @@ broker env fieldRef: fieldPath: metadata.name {{- if and .Values.broker.external.enabled (eq .Values.broker.external.type "NodePort") }} -- name: KAFKA_EXTERNAL_NODEPORT - value: "{{ include "broker.external.nodePort" . }}" +- name: KAFKA_EXTERNAL_NODEPORTS + value: "{{ include "broker.fullNodePorts" . }}" {{- end }} - name: KAFKA_HEAP_OPTS value: {{ .Values.broker.heapOpts | quote }} From 031863d0a3aa14cfcdcd083f5ec9ba006f7bee25 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 03:08:09 +0800 Subject: [PATCH 12/40] =?UTF-8?q?external=20nodePort=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/_helpers.tpl | 7 ++----- test-fixed-config.ps1 | 0 2 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 test-fixed-config.ps1 diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index d84201e..62642e9 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -89,7 +89,7 @@ broker.advertisedListeners.external {{- if .enabled -}} {{- if eq .type "NodePort" -}} {{- $addr = "$(POD_HOST_IP)" -}} - {{- $port = "$(echo $KAFKA_EXTERNAL_NODEPORTS | cut -d',' -f$(($(echo $POD_NAME | sed 's/.*-//')+1)))" -}} + {{- $port = "9999" -}} {{- else if eq .type "HostPort" -}} {{- $addr = "$(POD_HOST_IP)" -}} {{- $port = .hostPort | default .containerPort -}} @@ -137,10 +137,7 @@ broker env valueFrom: fieldRef: fieldPath: metadata.name -{{- if and .Values.broker.external.enabled (eq .Values.broker.external.type "NodePort") }} -- name: KAFKA_EXTERNAL_NODEPORTS - value: "{{ include "broker.fullNodePorts" . }}" -{{- end }} + - name: KAFKA_HEAP_OPTS value: {{ .Values.broker.heapOpts | quote }} - name: KAFKA_CFG_PROCESS_ROLES diff --git a/test-fixed-config.ps1 b/test-fixed-config.ps1 new file mode 100644 index 0000000..e69de29 From b9b79b18835f4603c7a846e711ee701b50bfd17d Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 03:15:27 +0800 Subject: [PATCH 13/40] =?UTF-8?q?external=20nodePort=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/templates/broker/cm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kafka/templates/broker/cm.yaml b/charts/kafka/templates/broker/cm.yaml index 0d059ae..d5e1b81 100644 --- a/charts/kafka/templates/broker/cm.yaml +++ b/charts/kafka/templates/broker/cm.yaml @@ -27,14 +27,14 @@ data: fix_external_advertised_port() { if [ -n "$1" ]; then local ext_port="$1" - export KAFKA_CFG_ADVERTISED_LISTENERS=$(echo "$KAFKA_CFG_ADVERTISED_LISTENERS" | sed -E 's%(EXTERNAL://.+?):[0-9]+%\1:'${ext_port}'%') + export KAFKA_CFG_ADVERTISED_LISTENERS=$(echo "$KAFKA_CFG_ADVERTISED_LISTENERS" | sed -E 's%(EXTERNAL://[^:]+):[0-9]+%\1:'${ext_port}'%') echo "KAFKA_CFG_ADVERTISED_LISTENERS: $KAFKA_CFG_ADVERTISED_LISTENERS" fi } fix_external_advertised_host() { if [ -n "$1" ]; then local ext_host="$1" - export KAFKA_CFG_ADVERTISED_LISTENERS=$(echo "$KAFKA_CFG_ADVERTISED_LISTENERS" | sed -E 's%EXTERNAL://.+?:([0-9]+)%EXTERNAL://'${ext_host}':\1%') + export KAFKA_CFG_ADVERTISED_LISTENERS=$(echo "$KAFKA_CFG_ADVERTISED_LISTENERS" | sed -E 's%EXTERNAL://[^:]+:([0-9]+)%EXTERNAL://'${ext_host}':\1%') echo "KAFKA_CFG_ADVERTISED_LISTENERS: $KAFKA_CFG_ADVERTISED_LISTENERS" fi } From feecab234de4473f551653238fff62c9c9fc1efa Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 10:38:39 +0800 Subject: [PATCH 14/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/broker/_helpers.tpl | 16 ++++++ .../templates/broker/cm-client-config.yaml | 13 +++++ charts/kafka-ha/templates/broker/cm-jaas.yaml | 23 +++++++++ .../templates/broker/job-init-scram.yaml | 51 +++++++++++++++++++ .../templates/broker/secret-sasl.yaml | 15 ++++++ .../templates/broker/statefulset.yaml | 19 +++++++ charts/kafka-ha/values.yaml | 10 ++++ 7 files changed, 147 insertions(+) create mode 100644 charts/kafka-ha/templates/broker/cm-client-config.yaml create mode 100644 charts/kafka-ha/templates/broker/cm-jaas.yaml create mode 100644 charts/kafka-ha/templates/broker/job-init-scram.yaml create mode 100644 charts/kafka-ha/templates/broker/secret-sasl.yaml diff --git a/charts/kafka-ha/templates/broker/_helpers.tpl b/charts/kafka-ha/templates/broker/_helpers.tpl index 77a00b1..1e7846c 100644 --- a/charts/kafka-ha/templates/broker/_helpers.tpl +++ b/charts/kafka-ha/templates/broker/_helpers.tpl @@ -136,9 +136,17 @@ broker env - name: KAFKA_CFG_ADVERTISED_LISTENERS value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + {{- if .Values.broker.auth.enabled }} + value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + {{- else }} value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + {{- end }} - name: KAFKA_CFG_INTER_BROKER_LISTENER_NAME + {{- if .Values.broker.auth.enabled }} + value: BROKER + {{- else }} value: BROKER + {{- end }} - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS @@ -165,6 +173,14 @@ broker env - name: KAFKA_NODE_ID_OFFSET value: "1000" {{- end }} +{{- if .Values.broker.auth.enabled }} +- name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server_jaas.conf" +- name: KAFKA_CFG_SASL_ENABLED_MECHANISMS + value: {{ .Values.broker.auth.mechanism | quote }} +- name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL + value: {{ .Values.broker.auth.mechanism | quote }} +{{- end }} {{- if .Values.broker.external.enabled -}} {{- include "broker.externalEnv" $ | nindent 0 }} {{- end }} diff --git a/charts/kafka-ha/templates/broker/cm-client-config.yaml b/charts/kafka-ha/templates/broker/cm-client-config.yaml new file mode 100644 index 0000000..864d2f6 --- /dev/null +++ b/charts/kafka-ha/templates/broker/cm-client-config.yaml @@ -0,0 +1,13 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-client-sasl-config + labels: + {{- include "kafka.labels" . | nindent 4 }} +data: + client.properties: | + security.protocol=SASL_PLAINTEXT + sasl.mechanism=SCRAM-SHA-256 + sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="{{ (index .Values.broker.auth.users 0).username }}" password="{{ (index .Values.broker.auth.users 0).password }}"; +{{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/broker/cm-jaas.yaml b/charts/kafka-ha/templates/broker/cm-jaas.yaml new file mode 100644 index 0000000..15fb2d0 --- /dev/null +++ b/charts/kafka-ha/templates/broker/cm-jaas.yaml @@ -0,0 +1,23 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kafka.broker.fullname" . }}-jaas + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: broker +data: + kafka_server_jaas.conf: | + KafkaServer { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.broker.auth.users 0).username }}" + password="{{ (index .Values.broker.auth.users 0).password }}"; + }; + + Client { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.broker.auth.users 0).username }}" + password="{{ (index .Values.broker.auth.users 0).password }}"; + }; +{{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/broker/job-init-scram.yaml b/charts/kafka-ha/templates/broker/job-init-scram.yaml new file mode 100644 index 0000000..566abaf --- /dev/null +++ b/charts/kafka-ha/templates/broker/job-init-scram.yaml @@ -0,0 +1,51 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "kafka.broker.fullname" . }}-init-scram + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: broker-init + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + {{- include "kafka.selectorLabels" $ | nindent 8 }} + app.kubernetes.io/component: broker-init + spec: + restartPolicy: OnFailure + containers: + - name: init-scram-users + image: {{ include "kafka.kafkaImage" $ | quote }} + {{- with $.Values.image.pullPolicy }} + imagePullPolicy: {{ . | quote }} + {{- end }} + command: + - /bin/bash + - -c + - | + echo "Waiting for Kafka to be ready..." + while ! bin/kafka-broker-api-versions.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ .Values.clusterDomain }}:9094 >/dev/null 2>&1; do + echo "Kafka not ready yet, waiting..." + sleep 10 + done + + echo "Kafka is ready, creating SCRAM users..." + + {{- range .Values.broker.auth.users }} + # Create SCRAM user: {{ .username }} + bin/kafka-configs.sh --bootstrap-server={{ include "kafka.broker.fullname" $ }}.{{ include "kafka.namespace" $ }}.svc.{{ $.Values.clusterDomain }}:9094 \ + --alter --add-config 'SCRAM-SHA-256=[password={{ .password }}]' \ + --entity-type users --entity-name {{ .username }} || true + {{- end }} + + echo "SCRAM users created successfully" + env: + - name: KAFKA_HEAP_OPTS + value: "-Xms256m -Xmx256m" +{{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/broker/secret-sasl.yaml b/charts/kafka-ha/templates/broker/secret-sasl.yaml new file mode 100644 index 0000000..c5619eb --- /dev/null +++ b/charts/kafka-ha/templates/broker/secret-sasl.yaml @@ -0,0 +1,15 @@ +{{- if .Values.broker.auth.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "kafka.broker.fullname" . }}-sasl + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: broker +type: Opaque +data: + {{- range .Values.broker.auth.users }} + {{ .username }}: {{ .password | b64enc | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/broker/statefulset.yaml b/charts/kafka-ha/templates/broker/statefulset.yaml index af3d54d..9948c94 100644 --- a/charts/kafka-ha/templates/broker/statefulset.yaml +++ b/charts/kafka-ha/templates/broker/statefulset.yaml @@ -98,6 +98,15 @@ spec: - mountPath: /entrypoint.sh name: entrypoint-sh subPath: entrypoint.sh + {{- if $.Values.broker.auth.enabled }} + - mountPath: /opt/kafka/config/client.properties + name: client-config + subPath: client.properties + {{- end }} + {{- if $.Values.broker.auth.enabled }} + - mountPath: "/etc/kafka/jaas" + name: jaas-config + {{- end }} {{- if eq (include "broker.external.autoDiscovery.enabled" $) "true" }} - name: auto-discovery-shared mountPath: /init-shared @@ -118,6 +127,16 @@ spec: path: entrypoint.sh name: {{ include "kafka.entrypoint.configmapName" $ }} defaultMode: 0744 + {{- if $.Values.broker.auth.enabled }} + - name: jaas-config + configMap: + name: {{ include "kafka.broker.fullname" $ }}-jaas + defaultMode: 0644 + - name: client-config + configMap: + name: kafka-client-sasl-config + defaultMode: 0644 + {{- end }} {{- if eq (include "broker.external.autoDiscovery.enabled" $) "true" }} - name: auto-discovery-shared emptyDir: {} diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 1111061..1975cc2 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -48,6 +48,16 @@ broker: heapOpts: "-Xms1024m -Xmx1024m" commonEnvs: [] extraEnvs: [] + + ## SASL SCRAM 认证配置 + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + # SASL用户凭据将通过Secret管理 + users: + - username: "kafka" + password: "Kafka@2025" + ## broker.config - server.properties 配置 ## 某些关键配置会被环境变量覆盖，例如: node.id advertised.listeners controller.quorum.voters 等 config: From d2661455b3e35ca02b5ec44471e50868caaf616c Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 12:27:02 +0800 Subject: [PATCH 15/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/values.yaml | 6 ++++++ test-fixed-config.ps1 | 0 2 files changed, 6 insertions(+) delete mode 100644 test-fixed-config.ps1 diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 1975cc2..2933d0b 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -140,6 +140,10 @@ controller: heapOpts: "-Xms512m -Xmx512m" commonEnvs: [] extraEnvs: [] + ## controller.labels + labels: {} + ## controller.annotations + annotations: {} persistence: enabled: true size: 20Gi @@ -174,6 +178,8 @@ controller: tolerations: [] ## controller.podAnnotations podAnnotations: {} + ## controller.podLabels + podLabels: {} ## controller.antiAffinity antiAffinity: soft antiAffinityTopologyKey: kubernetes.io/hostname diff --git a/test-fixed-config.ps1 b/test-fixed-config.ps1 deleted file mode 100644 index e69de29..0000000 From cafa9be310b70af4c07c9d855b38bc373952589b Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 13:53:10 +0800 Subject: [PATCH 16/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/controller/_helpers.tpl | 2 ++ charts/kafka-ha/templates/controller/statefulset.yaml | 2 +- charts/kafka-ha/values.yaml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index 624b097..1660093 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -17,6 +17,8 @@ controller.containerEnv value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} +- name: KAFKA_CFG_LOG_DIR + value: {{ .Values.controller.persistence.mountPath | quote }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index 4814943..f7426ce 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: /opt/kafka/data + - mountPath: {{ .Values.controller.persistence.mountPath }} name: data subPath: data - mountPath: /opt/kafka/logs diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 2933d0b..588c98a 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -147,6 +147,7 @@ controller: persistence: enabled: true size: 20Gi + mountPath: /opt/kafka/data # storageClass: "gp2" ## controller.resources resources: From b84de73e052f492eea941d43175adb751d33b0af Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 14:17:51 +0800 Subject: [PATCH 17/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/controller/_helpers.tpl | 2 +- charts/kafka-ha/templates/controller/statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index 1660093..30bbce6 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -18,7 +18,7 @@ controller.containerEnv - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} - name: KAFKA_CFG_LOG_DIR - value: {{ .Values.controller.persistence.mountPath | quote }} + value: {{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index f7426ce..f5cddeb 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: {{ .Values.controller.persistence.mountPath }} + - mountPath: {{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }} name: data subPath: data - mountPath: /opt/kafka/logs From eb3a78f4a12aaaa21951ea5c9478ad73b26641e2 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 14:34:26 +0800 Subject: [PATCH 18/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/controller/_helpers.tpl | 2 +- charts/kafka-ha/templates/controller/statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index 30bbce6..14ee07e 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -18,7 +18,7 @@ controller.containerEnv - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} - name: KAFKA_CFG_LOG_DIR - value: {{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }} + value: {{ if .Values.controller.persistence }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index f5cddeb..0d4a07a 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: {{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }} + - mountPath: {{ if .Values.controller.persistence }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} name: data subPath: data - mountPath: /opt/kafka/logs From 832caa625e385806c265579804fd7f6f9548bcaf Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 15:02:18 +0800 Subject: [PATCH 19/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/controller/_helpers.tpl | 2 +- charts/kafka-ha/templates/controller/statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index 14ee07e..7dd959f 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -18,7 +18,7 @@ controller.containerEnv - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} - name: KAFKA_CFG_LOG_DIR - value: {{ if .Values.controller.persistence }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} + value: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index 0d4a07a..c13c297 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: {{ if .Values.controller.persistence }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} + - mountPath: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} name: data subPath: data - mountPath: /opt/kafka/logs From ec3454169b10790e94112824d3d21a592e3f9804 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 15:07:47 +0800 Subject: [PATCH 20/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/controller/_helpers.tpl | 10 +- .../templates/controller/statefulset.yaml | 4 +- controller-nil-fix-final.md | 118 ++++++++++++++++++ 3 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 controller-nil-fix-final.md diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index 7dd959f..bd2c8fb 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -8,17 +8,17 @@ controller.containerEnv fieldRef: fieldPath: metadata.name - name: KAFKA_HEAP_OPTS - value: {{ .Values.controller.heapOpts | quote }} + value: {{ if .Values.controller }}{{ .Values.controller.heapOpts | quote }}{{ else }}"-Xmx1G -Xms1G"{{ end }} - name: KAFKA_CFG_PROCESS_ROLES value: controller - name: KAFKA_CFG_LISTENERS - value: "CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" + value: "CONTROLLER://0.0.0.0:{{ if .Values.controller }}{{ .Values.controller.containerPort }}{{ else }}9093{{ end }}" - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} - name: KAFKA_CFG_LOG_DIR - value: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} + value: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: @@ -26,7 +26,7 @@ controller.containerEnv key: clusterId - name: KAFKA_NODE_ID value: "podnameSuffix" -{{- with .Values.controller.extraEnvs }} - {{- toYaml . | nindent 0 }} +{{- if and .Values.controller .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 0 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index c13c297..c8068ee 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -61,7 +61,7 @@ spec: env: {{- include "controller.containerEnv" $ | nindent 8 }} ports: - - containerPort: {{ $.Values.controller.containerPort }} + - containerPort: {{ if $.Values.controller }}{{ $.Values.controller.containerPort }}{{ else }}9093{{ end }} name: controller protocol: TCP {{- with .livenessProbe }} @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} + - mountPath: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} name: data subPath: data - mountPath: /opt/kafka/logs diff --git a/controller-nil-fix-final.md b/controller-nil-fix-final.md new file mode 100644 index 0000000..b4ba4c7 --- /dev/null +++ b/controller-nil-fix-final.md @@ -0,0 +1,118 @@ +# Controller Nil Pointer 修复总结 + +## 问题描述 +使用 `--set controller.persistence.enabled="false"` 时，Helm 模板出现 nil pointer 错误： +``` +Error: INSTALLATION FAILED: template: kafka-ha/templates/controller/statefulset.yaml:84:41: executing "kafka-ha/templates/controller/statefulset.yaml" at <.Values.controller>: nil pointer evaluating interface {}.controller +``` + +## 根本原因 +当使用 `--set` 禁用某些配置时，Helm 可能完全移除整个 `controller` 对象，而不仅仅是设置 `enabled: false`。这导致模板中所有直接访问 `.Values.controller.*` 的地方都会出现 nil pointer 错误。 + +## 修复方案 + +### 1. StatefulSet 模板修复 +**文件**: `charts/kafka-ha/templates/controller/statefulset.yaml` + +**第64行 - containerPort**: +```yaml +# 修复前 +- containerPort: {{ $.Values.controller.containerPort }} + +# 修复后 +- containerPort: {{ if $.Values.controller }}{{ $.Values.controller.containerPort }}{{ else }}9093{{ end }} +``` + +**第84行 - mountPath**: +```yaml +# 修复前 +- mountPath: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} + +# 修复后 +- mountPath: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} +``` + +### 2. Helpers 模板修复 +**文件**: `charts/kafka-ha/templates/controller/_helpers.tpl` + +**KAFKA_HEAP_OPTS**: +```yaml +# 修复前 +value: {{ .Values.controller.heapOpts | quote }} + +# 修复后 +value: {{ if .Values.controller }}{{ .Values.controller.heapOpts | quote }}{{ else }}"-Xmx1G -Xms1G"{{ end }} +``` + +**KAFKA_CFG_LISTENERS**: +```yaml +# 修复前 +value: "CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" + +# 修复后 +value: "CONTROLLER://0.0.0.0:{{ if .Values.controller }}{{ .Values.controller.containerPort }}{{ else }}9093{{ end }}" +``` + +**KAFKA_CFG_LOG_DIR**: +```yaml +# 修复前 +value: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} + +# 修复后 +value: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} +``` + +**extraEnvs**: +```yaml +# 修复前 +{{- with .Values.controller.extraEnvs }} + {{- toYaml . | nindent 0 }} +{{- end }} + +# 修复后 +{{- if and .Values.controller .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 0 }} +{{- end }} +``` + +## 修复模式 + +### 1. 简单属性访问 +```yaml +{{ if .Values.controller }}{{ .Values.controller.property }}{{ else }}default_value{{ end }} +``` + +### 2. 嵌套对象访问 +```yaml +{{ if and .Values.controller (hasKey .Values.controller "nested_key") }}{{ .Values.controller.nested_key.property }}{{ else }}default_value{{ end }} +``` + +## 预期行为 + +### 场景1: 正常配置 +- `controller` 对象存在且完整 +- 使用配置的值或默认值 + +### 场景2: 部分禁用 (persistence.enabled=false) +- `controller` 对象存在但 `persistence.enabled=false` +- 仍然可以访问其他 controller 属性 + +### 场景3: 完全移除 controller 配置 +- `controller` 对象不存在 +- 使用硬编码的默认值 + +## 技术优势 + +1. **防御性编程**: 在访问对象属性前先检查对象是否存在 +2. **向后兼容**: 不影响正常的配置使用 +3. **优雅降级**: 提供合理的默认值 +4. **错误预防**: 避免 nil pointer 异常 + +## 测试命令 +```bash +# 现在应该可以正常工作 +helm install kafka-ha --set broker.persistence.enabled="false" --set controller.persistence.enabled="false" . +``` + +## 相关问题修复 +这个修复解决了所有因 `controller` 对象不存在而导致的 nil pointer 错误，确保 Helm chart 在各种配置场景下都能正常工作。 \ No newline at end of file From d79e16a326973f96034caaa301cd8bd71a4c2981 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 15:53:17 +0800 Subject: [PATCH 21/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/controller/_helpers.tpl | 10 +- .../templates/controller/statefulset.yaml | 4 +- charts/kafka-ha/values.yaml | 1 - controller-nil-fix-final.md | 118 ------------------ 4 files changed, 6 insertions(+), 127 deletions(-) delete mode 100644 controller-nil-fix-final.md diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index bd2c8fb..624b097 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -8,17 +8,15 @@ controller.containerEnv fieldRef: fieldPath: metadata.name - name: KAFKA_HEAP_OPTS - value: {{ if .Values.controller }}{{ .Values.controller.heapOpts | quote }}{{ else }}"-Xmx1G -Xms1G"{{ end }} + value: {{ .Values.controller.heapOpts | quote }} - name: KAFKA_CFG_PROCESS_ROLES value: controller - name: KAFKA_CFG_LISTENERS - value: "CONTROLLER://0.0.0.0:{{ if .Values.controller }}{{ .Values.controller.containerPort }}{{ else }}9093{{ end }}" + value: "CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} -- name: KAFKA_CFG_LOG_DIR - value: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: @@ -26,7 +24,7 @@ controller.containerEnv key: clusterId - name: KAFKA_NODE_ID value: "podnameSuffix" -{{- if and .Values.controller .Values.controller.extraEnvs }} - {{- toYaml .Values.controller.extraEnvs | nindent 0 }} +{{- with .Values.controller.extraEnvs }} + {{- toYaml . | nindent 0 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index c8068ee..4814943 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -61,7 +61,7 @@ spec: env: {{- include "controller.containerEnv" $ | nindent 8 }} ports: - - containerPort: {{ if $.Values.controller }}{{ $.Values.controller.containerPort }}{{ else }}9093{{ end }} + - containerPort: {{ $.Values.controller.containerPort }} name: controller protocol: TCP {{- with .livenessProbe }} @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} + - mountPath: /opt/kafka/data name: data subPath: data - mountPath: /opt/kafka/logs diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 588c98a..2933d0b 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -147,7 +147,6 @@ controller: persistence: enabled: true size: 20Gi - mountPath: /opt/kafka/data # storageClass: "gp2" ## controller.resources resources: diff --git a/controller-nil-fix-final.md b/controller-nil-fix-final.md deleted file mode 100644 index b4ba4c7..0000000 --- a/controller-nil-fix-final.md +++ /dev/null @@ -1,118 +0,0 @@ -# Controller Nil Pointer 修复总结 - -## 问题描述 -使用 `--set controller.persistence.enabled="false"` 时，Helm 模板出现 nil pointer 错误： -``` -Error: INSTALLATION FAILED: template: kafka-ha/templates/controller/statefulset.yaml:84:41: executing "kafka-ha/templates/controller/statefulset.yaml" at <.Values.controller>: nil pointer evaluating interface {}.controller -``` - -## 根本原因 -当使用 `--set` 禁用某些配置时，Helm 可能完全移除整个 `controller` 对象，而不仅仅是设置 `enabled: false`。这导致模板中所有直接访问 `.Values.controller.*` 的地方都会出现 nil pointer 错误。 - -## 修复方案 - -### 1. StatefulSet 模板修复 -**文件**: `charts/kafka-ha/templates/controller/statefulset.yaml` - -**第64行 - containerPort**: -```yaml -# 修复前 -- containerPort: {{ $.Values.controller.containerPort }} - -# 修复后 -- containerPort: {{ if $.Values.controller }}{{ $.Values.controller.containerPort }}{{ else }}9093{{ end }} -``` - -**第84行 - mountPath**: -```yaml -# 修复前 -- mountPath: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} - -# 修复后 -- mountPath: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" }}{{ else }}"/opt/kafka/data"{{ end }} -``` - -### 2. Helpers 模板修复 -**文件**: `charts/kafka-ha/templates/controller/_helpers.tpl` - -**KAFKA_HEAP_OPTS**: -```yaml -# 修复前 -value: {{ .Values.controller.heapOpts | quote }} - -# 修复后 -value: {{ if .Values.controller }}{{ .Values.controller.heapOpts | quote }}{{ else }}"-Xmx1G -Xms1G"{{ end }} -``` - -**KAFKA_CFG_LISTENERS**: -```yaml -# 修复前 -value: "CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" - -# 修复后 -value: "CONTROLLER://0.0.0.0:{{ if .Values.controller }}{{ .Values.controller.containerPort }}{{ else }}9093{{ end }}" -``` - -**KAFKA_CFG_LOG_DIR**: -```yaml -# 修复前 -value: {{ if hasKey .Values.controller "persistence" }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} - -# 修复后 -value: {{ if and .Values.controller (hasKey .Values.controller "persistence") }}{{ .Values.controller.persistence.mountPath | default "/opt/kafka/data" | quote }}{{ else }}"/opt/kafka/data"{{ end }} -``` - -**extraEnvs**: -```yaml -# 修复前 -{{- with .Values.controller.extraEnvs }} - {{- toYaml . | nindent 0 }} -{{- end }} - -# 修复后 -{{- if and .Values.controller .Values.controller.extraEnvs }} - {{- toYaml .Values.controller.extraEnvs | nindent 0 }} -{{- end }} -``` - -## 修复模式 - -### 1. 简单属性访问 -```yaml -{{ if .Values.controller }}{{ .Values.controller.property }}{{ else }}default_value{{ end }} -``` - -### 2. 嵌套对象访问 -```yaml -{{ if and .Values.controller (hasKey .Values.controller "nested_key") }}{{ .Values.controller.nested_key.property }}{{ else }}default_value{{ end }} -``` - -## 预期行为 - -### 场景1: 正常配置 -- `controller` 对象存在且完整 -- 使用配置的值或默认值 - -### 场景2: 部分禁用 (persistence.enabled=false) -- `controller` 对象存在但 `persistence.enabled=false` -- 仍然可以访问其他 controller 属性 - -### 场景3: 完全移除 controller 配置 -- `controller` 对象不存在 -- 使用硬编码的默认值 - -## 技术优势 - -1. **防御性编程**: 在访问对象属性前先检查对象是否存在 -2. **向后兼容**: 不影响正常的配置使用 -3. **优雅降级**: 提供合理的默认值 -4. **错误预防**: 避免 nil pointer 异常 - -## 测试命令 -```bash -# 现在应该可以正常工作 -helm install kafka-ha --set broker.persistence.enabled="false" --set controller.persistence.enabled="false" . -``` - -## 相关问题修复 -这个修复解决了所有因 `controller` 对象不存在而导致的 nil pointer 错误，确保 Helm chart 在各种配置场景下都能正常工作。 \ No newline at end of file From fcb14bcbd94206f0f19553f4e0650aa503ea491f Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 17:23:35 +0800 Subject: [PATCH 22/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/cm-entrypoint.yaml | 1 + test-storage-fix.ps1 | 69 ++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 test-storage-fix.ps1 diff --git a/charts/kafka-ha/templates/cm-entrypoint.yaml b/charts/kafka-ha/templates/cm-entrypoint.yaml index aa7c669..a92fbe3 100644 --- a/charts/kafka-ha/templates/cm-entrypoint.yaml +++ b/charts/kafka-ha/templates/cm-entrypoint.yaml @@ -19,6 +19,7 @@ data: export KAFKA_BASE_CONF_FILE="${KAFKA_BASE_CONF_FILE:=/etc/kafka/base-cm/server.properties}" export KAFKA_CFG_LOG_DIR="${KAFKA_CFG_LOG_DIR:=/var/lib/kafka/data}" + export KAFKA_CFG_LOG_DIRS="${KAFKA_CFG_LOG_DIRS:=$KAFKA_CFG_LOG_DIR}" check_runtime() { java -version diff --git a/test-storage-fix.ps1 b/test-storage-fix.ps1 new file mode 100644 index 0000000..a986960 --- /dev/null +++ b/test-storage-fix.ps1 @@ -0,0 +1,69 @@ +#!/usr/bin/env pwsh + +Write-Host "=== Kafka Storage Path Fix Validation ===" -ForegroundColor Green + +# 1. Check if KAFKA_CFG_LOG_DIRS is set in entrypoint +Write-Host "`n1. Checking entrypoint script for LOG_DIRS configuration..." -ForegroundColor Yellow +$entrypointFile = "charts\kafka-ha\templates\cm-entrypoint.yaml" +if (Test-Path $entrypointFile) { + $logDirConfig = Select-String -Path $entrypointFile -Pattern "KAFKA_CFG_LOG_DIR" + if ($logDirConfig) { + Write-Host "OK Found LOG_DIR configurations:" -ForegroundColor Green + $logDirConfig | ForEach-Object { Write-Host " $($_.Line.Trim())" -ForegroundColor Cyan } + } else { + Write-Host "ERROR No LOG_DIR configuration found" -ForegroundColor Red + exit 1 + } +} else { + Write-Host "ERROR Entrypoint file not found" -ForegroundColor Red + exit 1 +} + +# 2. Check persistence configuration +Write-Host "`n2. Checking persistence configuration..." -ForegroundColor Yellow +$valuesFile = "charts\kafka-ha\values.yaml" +if (Test-Path $valuesFile) { + $persistenceConfig = Select-String -Path $valuesFile -Pattern "mountPath.*kafka" + if ($persistenceConfig) { + Write-Host "OK Found persistence mount path:" -ForegroundColor Green + $persistenceConfig | ForEach-Object { Write-Host " $($_.Line.Trim())" -ForegroundColor Cyan } + } else { + Write-Host "ERROR No persistence mount path found" -ForegroundColor Red + exit 1 + } +} else { + Write-Host "ERROR Values file not found" -ForegroundColor Red + exit 1 +} + +# 3. Check volume mount in statefulset +Write-Host "`n3. Checking volume mount in statefulset..." -ForegroundColor Yellow +$statefulsetFile = "charts\kafka-ha\templates\broker\statefulset.yaml" +if (Test-Path $statefulsetFile) { + $volumeMount = Select-String -Path $statefulsetFile -Pattern "mountPath.*persistence\.mountPath" + if ($volumeMount) { + Write-Host "OK Found volume mount configuration:" -ForegroundColor Green + $volumeMount | ForEach-Object { Write-Host " $($_.Line.Trim())" -ForegroundColor Cyan } + } else { + Write-Host "ERROR No volume mount configuration found" -ForegroundColor Red + exit 1 + } +} else { + Write-Host "ERROR Statefulset file not found" -ForegroundColor Red + exit 1 +} + +# 4. Check storage format logic +Write-Host "`n4. Checking storage format logic..." -ForegroundColor Yellow +$formatLogic = Select-String -Path $entrypointFile -Pattern "init_storage_format_if_needed|meta\.properties" +if ($formatLogic) { + Write-Host "OK Found storage format logic:" -ForegroundColor Green + $formatLogic | ForEach-Object { Write-Host " Line $($_.LineNumber): $($_.Line.Trim())" -ForegroundColor Cyan } +} else { + Write-Host "ERROR No storage format logic found" -ForegroundColor Red + exit 1 +} + +Write-Host "`n=== Validation completed ===" -ForegroundColor Green +Write-Host "The storage path configuration should now work correctly." -ForegroundColor Green +Write-Host "Kafka will use the default path /var/lib/kafka/data for log.dirs" -ForegroundColor Green \ No newline at end of file From 964d0b39a36313d88dd83fa40bad129b2a49ec2e Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 17:23:49 +0800 Subject: [PATCH 23/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test-storage-fix.ps1 | 69 -------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 test-storage-fix.ps1 diff --git a/test-storage-fix.ps1 b/test-storage-fix.ps1 deleted file mode 100644 index a986960..0000000 --- a/test-storage-fix.ps1 +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env pwsh - -Write-Host "=== Kafka Storage Path Fix Validation ===" -ForegroundColor Green - -# 1. Check if KAFKA_CFG_LOG_DIRS is set in entrypoint -Write-Host "`n1. Checking entrypoint script for LOG_DIRS configuration..." -ForegroundColor Yellow -$entrypointFile = "charts\kafka-ha\templates\cm-entrypoint.yaml" -if (Test-Path $entrypointFile) { - $logDirConfig = Select-String -Path $entrypointFile -Pattern "KAFKA_CFG_LOG_DIR" - if ($logDirConfig) { - Write-Host "OK Found LOG_DIR configurations:" -ForegroundColor Green - $logDirConfig | ForEach-Object { Write-Host " $($_.Line.Trim())" -ForegroundColor Cyan } - } else { - Write-Host "ERROR No LOG_DIR configuration found" -ForegroundColor Red - exit 1 - } -} else { - Write-Host "ERROR Entrypoint file not found" -ForegroundColor Red - exit 1 -} - -# 2. Check persistence configuration -Write-Host "`n2. Checking persistence configuration..." -ForegroundColor Yellow -$valuesFile = "charts\kafka-ha\values.yaml" -if (Test-Path $valuesFile) { - $persistenceConfig = Select-String -Path $valuesFile -Pattern "mountPath.*kafka" - if ($persistenceConfig) { - Write-Host "OK Found persistence mount path:" -ForegroundColor Green - $persistenceConfig | ForEach-Object { Write-Host " $($_.Line.Trim())" -ForegroundColor Cyan } - } else { - Write-Host "ERROR No persistence mount path found" -ForegroundColor Red - exit 1 - } -} else { - Write-Host "ERROR Values file not found" -ForegroundColor Red - exit 1 -} - -# 3. Check volume mount in statefulset -Write-Host "`n3. Checking volume mount in statefulset..." -ForegroundColor Yellow -$statefulsetFile = "charts\kafka-ha\templates\broker\statefulset.yaml" -if (Test-Path $statefulsetFile) { - $volumeMount = Select-String -Path $statefulsetFile -Pattern "mountPath.*persistence\.mountPath" - if ($volumeMount) { - Write-Host "OK Found volume mount configuration:" -ForegroundColor Green - $volumeMount | ForEach-Object { Write-Host " $($_.Line.Trim())" -ForegroundColor Cyan } - } else { - Write-Host "ERROR No volume mount configuration found" -ForegroundColor Red - exit 1 - } -} else { - Write-Host "ERROR Statefulset file not found" -ForegroundColor Red - exit 1 -} - -# 4. Check storage format logic -Write-Host "`n4. Checking storage format logic..." -ForegroundColor Yellow -$formatLogic = Select-String -Path $entrypointFile -Pattern "init_storage_format_if_needed|meta\.properties" -if ($formatLogic) { - Write-Host "OK Found storage format logic:" -ForegroundColor Green - $formatLogic | ForEach-Object { Write-Host " Line $($_.LineNumber): $($_.Line.Trim())" -ForegroundColor Cyan } -} else { - Write-Host "ERROR No storage format logic found" -ForegroundColor Red - exit 1 -} - -Write-Host "`n=== Validation completed ===" -ForegroundColor Green -Write-Host "The storage path configuration should now work correctly." -ForegroundColor Green -Write-Host "Kafka will use the default path /var/lib/kafka/data for log.dirs" -ForegroundColor Green \ No newline at end of file From 11f950f7a288139085e2c8c0925b4db04cc8b779 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 17:37:04 +0800 Subject: [PATCH 24/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/cm-entrypoint.yaml | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/charts/kafka-ha/templates/cm-entrypoint.yaml b/charts/kafka-ha/templates/cm-entrypoint.yaml index a92fbe3..796b60a 100644 --- a/charts/kafka-ha/templates/cm-entrypoint.yaml +++ b/charts/kafka-ha/templates/cm-entrypoint.yaml @@ -170,12 +170,32 @@ data: take_logdir_ownership_if_needed() { local dir="$1" + # Create directory if it doesn't exist + if [ ! -d "$dir" ]; then + echo "Creating directory: $dir" + mkdir -p "$dir" + fi + if [ -d "$dir" ]; then if [ "$KAFKA_LOGDIR_CHOWN_FORCE" = "true" ]; then + # Also ensure parent directories have correct ownership + parent_dir=$(dirname "$dir") + if [ -d "$parent_dir" ] && [ "$parent_dir" != "/" ]; then + chown -R 1000:1000 "$parent_dir" + chmod -R 755 "$parent_dir" + fi chown -R 1000:1000 "$dir" + chmod -R 755 "$dir" echo "ls -alh $dir" ; ls -alh "$dir" elif [ "$(stat -c "%u" $dir)" != "1000" ]; then + # Also ensure parent directories have correct ownership + parent_dir=$(dirname "$dir") + if [ -d "$parent_dir" ] && [ "$parent_dir" != "/" ]; then + chown -R 1000:1000 "$parent_dir" + chmod -R 755 "$parent_dir" + fi chown -R 1000:1000 "$dir" + chmod -R 755 "$dir" echo "ls -alh $dir" ; ls -alh "$dir" fi fi @@ -204,6 +224,23 @@ data: if [ -n "$KAFKA_CFG_LOG_DIRS" ]; then logdir=$(echo "$KAFKA_CFG_LOG_DIRS" | cut -d "," -f 1) fi + + # Ensure log directory and parent directories exist with correct permissions + if [ ! -d "$logdir" ]; then + echo "Creating log directory: $logdir" + mkdir -p "$logdir" + if [ "$(id -u)" = "0" ]; then + # Also ensure parent directories have correct ownership + parent_dir=$(dirname "$logdir") + if [ -d "$parent_dir" ] && [ "$parent_dir" != "/" ]; then + chown -R 1000:1000 "$parent_dir" + chmod -R 755 "$parent_dir" + fi + chown -R 1000:1000 "$logdir" + chmod -R 755 "$logdir" + fi + fi + if [ ! -f "$logdir/meta.properties" ]; then echo ">>> Format Log Directories <<<" if [ -z "$KAFKA_CLUSTER_ID" ]; then From 6a8da9456dfb3180badfac73c08257246fe28ebc Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 17:50:52 +0800 Subject: [PATCH 25/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/broker/_helpers.tpl | 21 +++++++++---------- .../templates/broker/svc-headless.yaml | 4 ++++ charts/kafka-ha/templates/broker/svc.yaml | 4 ++++ charts/kafka-ha/values.yaml | 2 ++ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/charts/kafka-ha/templates/broker/_helpers.tpl b/charts/kafka-ha/templates/broker/_helpers.tpl index 1e7846c..d215eb4 100644 --- a/charts/kafka-ha/templates/broker/_helpers.tpl +++ b/charts/kafka-ha/templates/broker/_helpers.tpl @@ -5,6 +5,9 @@ broker.containerPorts - containerPort: {{ .Values.broker.containerPort }} name: broker protocol: TCP +- containerPort: {{ .Values.broker.internalPort | default 9094 }} + name: internal + protocol: TCP {{- with .Values.broker.external }} {{- if .enabled }} - containerPort: {{ .containerPort }} @@ -72,8 +75,8 @@ broker.advertisedListeners.internal */}} {{- define "broker.advertisedListeners.internal" -}} {{- $serviceAddr := (include "broker.headless.serviceAddr" .) -}} -{{- $port := .Values.broker.containerPort | int -}} -{{- printf "BROKER://$(POD_NAME).%s:%d" $serviceAddr $port -}} +{{- $port := .Values.broker.internalPort | default 9094 | int -}} +{{- printf "INTERNAL://$(POD_NAME).%s:%d" $serviceAddr $port -}} {{- end -}} {{/* @@ -129,24 +132,20 @@ broker env {{- end }} - name: KAFKA_CFG_LISTENERS {{- if not .Values.controller.enabled }} - value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }},CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" + value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},INTERNAL://0.0.0.0:{{ .Values.broker.internalPort | default 9094 }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }},CONTROLLER://0.0.0.0:{{ .Values.controller.containerPort }}" {{- else }} - value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }}" + value: "BROKER://0.0.0.0:{{ .Values.broker.containerPort }},INTERNAL://0.0.0.0:{{ .Values.broker.internalPort | default 9094 }},EXTERNAL://0.0.0.0:{{ .Values.broker.external.containerPort }}" {{- end }} - name: KAFKA_CFG_ADVERTISED_LISTENERS value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP {{- if .Values.broker.auth.enabled }} - value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT {{- else }} - value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT + value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT {{- end }} - name: KAFKA_CFG_INTER_BROKER_LISTENER_NAME - {{- if .Values.broker.auth.enabled }} - value: BROKER - {{- else }} - value: BROKER - {{- end }} + value: INTERNAL - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS diff --git a/charts/kafka-ha/templates/broker/svc-headless.yaml b/charts/kafka-ha/templates/broker/svc-headless.yaml index 8ffcb3a..945d4a4 100644 --- a/charts/kafka-ha/templates/broker/svc-headless.yaml +++ b/charts/kafka-ha/templates/broker/svc-headless.yaml @@ -38,6 +38,10 @@ spec: targetPort: broker protocol: TCP name: broker + - port: {{ $.Values.broker.internalPort | default 9094 }} + targetPort: internal + protocol: TCP + name: internal - port: {{ $.Values.controller.containerPort }} targetPort: controller protocol: TCP diff --git a/charts/kafka-ha/templates/broker/svc.yaml b/charts/kafka-ha/templates/broker/svc.yaml index d33de7f..48c0358 100644 --- a/charts/kafka-ha/templates/broker/svc.yaml +++ b/charts/kafka-ha/templates/broker/svc.yaml @@ -24,6 +24,10 @@ spec: port: {{ $.Values.broker.containerPort }} targetPort: broker protocol: TCP + - name: internal + port: {{ $.Values.broker.internalPort | default 9094 }} + targetPort: internal + protocol: TCP {{- if $.Values.broker.external.enabled }} - name: external port: {{ $.Values.broker.external.containerPort }} diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 2933d0b..52c696e 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -44,6 +44,8 @@ global: broker: replicaCount: 3 containerPort: 9092 + # Internal communication port for broker-to-broker communication (plaintext) + internalPort: 9094 terminationGracePeriodSeconds: 60 heapOpts: "-Xms1024m -Xmx1024m" commonEnvs: [] From ae507fc0b86c724e648a8320a1245930d4d7b468 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 18:18:56 +0800 Subject: [PATCH 26/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/broker/_helpers.tpl | 2 +- charts/kafka-ha/templates/broker/statefulset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kafka-ha/templates/broker/_helpers.tpl b/charts/kafka-ha/templates/broker/_helpers.tpl index d215eb4..609ef1c 100644 --- a/charts/kafka-ha/templates/broker/_helpers.tpl +++ b/charts/kafka-ha/templates/broker/_helpers.tpl @@ -243,7 +243,7 @@ broker.internal.bootstrapServers {{- define "broker.internal.bootstrapServers" -}} {{- $brokerFullName := include "kafka.broker.fullname" . -}} {{- $domainSuffix := (include "broker.headless.serviceAddr" .) -}} -{{- $brokerPort := .Values.broker.containerPort | int -}} +{{- $brokerPort := .Values.broker.internalPort | default 9094 | int -}} {{- $servers := list -}} {{- $brokerReplicaCount := int .Values.broker.replicaCount -}} {{- range $i := until $brokerReplicaCount -}} diff --git a/charts/kafka-ha/templates/broker/statefulset.yaml b/charts/kafka-ha/templates/broker/statefulset.yaml index 9948c94..cbf85e9 100644 --- a/charts/kafka-ha/templates/broker/statefulset.yaml +++ b/charts/kafka-ha/templates/broker/statefulset.yaml @@ -74,7 +74,7 @@ spec: command: - sh - -c - - bin/kafka-broker-api-versions.sh --bootstrap-server=127.0.0.1:{{ $.Values.broker.containerPort }} + - bin/kafka-broker-api-versions.sh --bootstrap-server=127.0.0.1:{{ $.Values.broker.internalPort | default 9094 }} {{- with .readinessProbe }} {{- toYaml . | nindent 10 }} {{- end }} From 33802b8612d5885f57c26ebef243c2f23d4f2bd8 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 18:31:09 +0800 Subject: [PATCH 27/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/broker/_helpers.tpl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/charts/kafka-ha/templates/broker/_helpers.tpl b/charts/kafka-ha/templates/broker/_helpers.tpl index 609ef1c..52c7cf5 100644 --- a/charts/kafka-ha/templates/broker/_helpers.tpl +++ b/charts/kafka-ha/templates/broker/_helpers.tpl @@ -79,6 +79,15 @@ broker.advertisedListeners.internal {{- printf "INTERNAL://$(POD_NAME).%s:%d" $serviceAddr $port -}} {{- end -}} +{{/* +broker.advertisedListeners.broker +*/}} +{{- define "broker.advertisedListeners.broker" -}} +{{- $serviceAddr := (include "broker.headless.serviceAddr" .) -}} +{{- $port := .Values.broker.containerPort | int -}} +{{- printf "BROKER://$(POD_NAME).%s:%d" $serviceAddr $port -}} +{{- end -}} + {{/* broker.advertisedListeners.external */}} @@ -103,7 +112,11 @@ broker.advertisedListeners.external broker.config.advertised.listeners */}} {{- define "broker.config.advertised.listeners" -}} -{{- printf "%s,%s" (include "broker.advertisedListeners.internal" .) (include "broker.advertisedListeners.external" .) -}} +{{- $listeners := list (include "broker.advertisedListeners.internal" .) (include "broker.advertisedListeners.broker" .) -}} +{{- if .Values.broker.external.enabled -}} +{{- $listeners = append $listeners (include "broker.advertisedListeners.external" .) -}} +{{- end -}} +{{- join "," $listeners -}} {{- end -}} {{/* From c0d3b4bd5f05e577294a81d5677954c34a9c1fe3 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 18:49:27 +0800 Subject: [PATCH 28/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-ha=20SASL?= =?UTF-8?q?=E8=AE=A4=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index b13ce08..9782a95 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -13,7 +13,7 @@ clusterId: "YLlmZWU5YTItYThjNi00Zg" image: repository: kafkace/kafka pullPolicy: IfNotPresent - tag: v3.7.1-63ba8d2 + tag: v4.0.0 rbac: create: true From 2fcdbd895e8cf52471399bdbbd2006cf49a46f21 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 18:58:15 +0800 Subject: [PATCH 29/40] =?UTF-8?q?=E4=BD=BF=E7=94=A8kafka=20SASL=20SCRAM?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/sasl-scram.md | 146 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 136 insertions(+), 10 deletions(-) diff --git a/docs/sasl-scram.md b/docs/sasl-scram.md index 69caee3..e162fc1 100644 --- a/docs/sasl-scram.md +++ b/docs/sasl-scram.md @@ -29,7 +29,51 @@ broker: - `sasl.enabled.mechanisms=SCRAM-SHA-256` - `sasl.mechanism.inter.broker.protocol=SCRAM-SHA-256` - `security.inter.broker.protocol=SASL_PLAINTEXT` -- `listener.security.protocol.map=CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT` +- `listener.security.protocol.map=CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT` + +### 3. 监听器端口配置 + +Kafka 集群配置了以下监听器： + +| 监听器名称 | 端口 | 协议 | 用途 | SASL认证 | +|-----------|------|------|------|----------| +| CONTROLLER | 9093 | PLAINTEXT | Controller间通信 | 否 | +| BROKER | 9092 | SASL_PLAINTEXT | 客户端连接 | 是 | +| INTERNAL | 9094 | PLAINTEXT | Broker间通信 | 否 | +| EXTERNAL | 9095 | SASL_PLAINTEXT | 外部客户端连接 | 是 | + +### 4. Advertised Listeners 配置 + +系统会自动配置 `advertised.listeners`，包含： + +- `INTERNAL://kafka-ha-broker-0.kafka-ha-headless.default.svc.cluster.local:9094` +- `BROKER://kafka-ha-broker-0.kafka-ha-headless.default.svc.cluster.local:9092` +- `EXTERNAL://...` (仅在 `broker.external.enabled=true` 时) + +**重要说明**：客户端应使用 **BROKER** 监听器（端口9092）进行连接，该监听器启用了SASL认证。 + +### 5. External 监听器配置 + +如果需要启用外部访问，可以配置： + +```yaml +broker: + external: + enabled: true # 启用外部监听器 + type: NodePort # 或 LoadBalancer + containerPort: 9095 +``` + +启用后，`advertised.listeners` 将包含 EXTERNAL 监听器，允许集群外部的客户端连接。 + +### 6. 监听器配置最佳实践 + +1. **内部通信**：INTERNAL 监听器用于 broker 间通信，无需认证 +2. **客户端连接**：BROKER 监听器用于客户端连接，启用 SASL 认证 +3. **外部访问**：EXTERNAL 监听器用于集群外部访问，启用 SASL 认证 +4. **Controller 通信**：CONTROLLER 监听器用于 controller 间通信，无需认证 + +**注意**：确保客户端使用正确的端口和协议进行连接，避免 "no matching listener" 错误。 ## 部署步骤 @@ -74,14 +118,14 @@ kubectl apply -f examples/kafka-client-sasl.yaml # 进入客户端 Pod kubectl exec -it kafka-client-sasl -- bash -# 创建主题 -bin/kafka-topics.sh --bootstrap-server kafka-sasl-broker:9092 \ +# 创建主题 (使用BROKER监听器端口9092) +bin/kafka-topics.sh --bootstrap-server kafka-ha-broker:9092 \ --command-config /etc/kafka/client/client.properties \ --create --topic test-topic --partitions 2 --replication-factor 1 # 发送消息 echo "Hello SASL SCRAM" | bin/kafka-console-producer.sh \ - --bootstrap-server kafka-sasl-broker:9092 \ + --bootstrap-server kafka-ha-broker:9092 \ --producer.config /etc/kafka/client/producer.properties \ --topic test-topic ``` @@ -91,7 +135,7 @@ echo "Hello SASL SCRAM" | bin/kafka-console-producer.sh \ ```bash # 消费消息 bin/kafka-console-consumer.sh \ - --bootstrap-server kafka-sasl-broker:9092 \ + --bootstrap-server kafka-ha-broker:9092 \ --consumer.config /etc/kafka/client/consumer.properties \ --topic test-topic --from-beginning ``` @@ -101,7 +145,8 @@ bin/kafka-console-consumer.sh \ ### Java 客户端配置 ```properties -bootstrap.servers=kafka-sasl:9092 +# 使用BROKER监听器端口9092进行SASL认证连接 +bootstrap.servers=kafka-ha-broker:9092 security.protocol=SASL_PLAINTEXT sasl.mechanism=SCRAM-SHA-256 sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; @@ -112,8 +157,9 @@ sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule require ```python from kafka import KafkaProducer, KafkaConsumer +# 使用BROKER监听器端口9092进行SASL认证连接 producer = KafkaProducer( - bootstrap_servers=['kafka-sasl:9092'], + bootstrap_servers=['kafka-ha-broker:9092'], security_protocol='SASL_PLAINTEXT', sasl_mechanism='SCRAM-SHA-256', sasl_plain_username='kafka', @@ -122,7 +168,7 @@ producer = KafkaProducer( consumer = KafkaConsumer( 'test-topic', - bootstrap_servers=['kafka-sasl:9092'], + bootstrap_servers=['kafka-ha-broker:9092'], security_protocol='SASL_PLAINTEXT', sasl_mechanism='SCRAM-SHA-256', sasl_plain_username='kafka', @@ -155,14 +201,54 @@ consumer = KafkaConsumer( - 网络连接是否正常 - 端口配置是否正确 -### 3. 查看详细日志 +### 3. "No matching listener" 错误 + +如果遇到 `partitions have leader brokers without a matching listener` 错误： + +**问题原因**：客户端无法找到匹配的监听器，通常是因为： +- 使用了错误的端口号 +- `advertised.listeners` 配置不完整 +- 监听器协议不匹配 + +**解决方案**： +1. **确认使用正确的端口**： + - SASL 客户端应使用端口 **9092** (BROKER 监听器) + - 不要使用端口 9094 (INTERNAL 监听器，仅用于 broker 间通信) + +2. **检查 advertised.listeners 配置**： + ```bash + # 查看当前配置 + kubectl exec kafka-ha-broker-0 -- env | grep KAFKA_CFG_ADVERTISED_LISTENERS + ``` + +3. **验证监听器配置**： + ```bash + # 应该包含 BROKER 监听器 + # INTERNAL://kafka-ha-broker-0.kafka-ha-headless.default.svc.cluster.local:9094, + # BROKER://kafka-ha-broker-0.kafka-ha-headless.default.svc.cluster.local:9092 + ``` + +### 4. 客户端连接配置检查 + +确保客户端配置正确： +```properties +# 正确的配置 +bootstrap.servers=kafka-ha-broker:9092 # 使用 BROKER 监听器 +security.protocol=SASL_PLAINTEXT +sasl.mechanism=SCRAM-SHA-256 +``` + +### 5. 查看详细日志 ```bash # 查看 Kafka broker 日志 kubectl logs -l app.kubernetes.io/component=broker -f # 查看 SCRAM 初始化 Job 日志 -kubectl logs job/kafka-sasl-broker-init-scram +kubectl logs job/kafka-ha-broker-init-scram + +# 检查监听器配置 +kubectl exec kafka-ha-broker-0 -- cat /opt/bitnami/kafka/config/server.properties | grep listeners ``` ## 升级和维护 @@ -186,6 +272,46 @@ helm upgrade kafka-sasl ./charts/kafka -f examples/values-sasl-scram.yml 2. 执行 Helm 升级 3. SCRAM 初始化 Job 会自动更新用户凭据 +## 快速参考 + +### 端口和监听器速查表 + +| 端口 | 监听器 | 协议 | 用途 | SASL认证 | 客户端使用 | +|------|--------|------|------|----------|------------| +| 9092 | BROKER | SASL_PLAINTEXT | 客户端连接 | ✅ | **推荐** | +| 9093 | CONTROLLER | PLAINTEXT | Controller通信 | ❌ | 禁止 | +| 9094 | INTERNAL | PLAINTEXT | Broker间通信 | ❌ | 禁止 | +| 9095 | EXTERNAL | SASL_PLAINTEXT | 外部客户端 | ✅ | 可选 | + +### 常用命令 + +```bash +# 部署 SASL Kafka +helm install kafka-ha charts/kafka-ha -f examples/values-sasl-scram.yml + +# 检查 Pod 状态 +kubectl get pods -l app.kubernetes.io/name=kafka-ha + +# 查看监听器配置 +kubectl exec kafka-ha-broker-0 -- env | grep KAFKA_CFG_ADVERTISED_LISTENERS + +# 测试连接 +kubectl exec -it kafka-client -- kafka-topics.sh \ + --bootstrap-server kafka-ha-broker:9092 \ + --command-config /etc/kafka/client.properties \ + --list +``` + +### 客户端配置模板 + +```properties +# 基本 SASL 配置 +bootstrap.servers=kafka-ha-broker:9092 +security.protocol=SASL_PLAINTEXT +sasl.mechanism=SCRAM-SHA-256 +sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="kafka" password="Kafka@2025"; +``` + ## 参考资料 - [Apache Kafka SASL/SCRAM 文档](https://kafka.apache.org/documentation/#security_sasl_scram) From 19cdb47017592933593cc83febd8e9f5040c353c Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 19:09:43 +0800 Subject: [PATCH 30/40] =?UTF-8?q?=E6=96=B0=E5=A2=9Ekafka=20SASL=20SCRAM?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 52c696e..d032435 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -10,7 +10,7 @@ clusterId: "YLlmZWU5YTItYThjNi00Zg" image: repository: kafkace/kafka pullPolicy: IfNotPresent - tag: v3.7.1-63ba8d2 + tag: v4.0.0 rbac: create: true serviceAccount: From 952a1895f67baecf2034304a4a71c969e7a77ee0 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 20:41:01 +0800 Subject: [PATCH 31/40] =?UTF-8?q?=E6=96=B0=E5=A2=9Ekafka=20SASL=20SCRAM?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/controller/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index 4814943..35d0e72 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -81,7 +81,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} volumeMounts: - - mountPath: /opt/kafka/data + - mountPath: /var/lib/kafka/data name: data subPath: data - mountPath: /opt/kafka/logs From 122a5031b4e04fe62bff939e2482915b260a1023 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Sun, 21 Sep 2025 20:47:19 +0800 Subject: [PATCH 32/40] =?UTF-8?q?=E5=A2=9E=E5=8A=A0kafka-ha=20pv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/pv-broker.yaml | 77 +++++++++++++++++++++++++++++++++++++ examples/pv-controller.yaml | 77 +++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 examples/pv-broker.yaml create mode 100644 examples/pv-controller.yaml diff --git a/examples/pv-broker.yaml b/examples/pv-broker.yaml new file mode 100644 index 0000000..2882dda --- /dev/null +++ b/examples/pv-broker.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-broker1 +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: + claimRef: + name: data-kafka-ha-broker-0 + namespace: default + local: + path: /data/broker + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master01 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-broker2 +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: + claimRef: + name: data-kafka-ha-broker-1 + namespace: default + local: + path: /data/broker + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master02 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-broker3 +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: + claimRef: + name: data-kafka-ha-broker-2 + namespace: default + local: + path: /data/broker + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master03 diff --git a/examples/pv-controller.yaml b/examples/pv-controller.yaml new file mode 100644 index 0000000..ccea9fc --- /dev/null +++ b/examples/pv-controller.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-controller1 +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: + claimRef: + name: data-kafka-ha-controller-0 + namespace: default + local: + path: /data/controller + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master01 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-controller2 +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: + claimRef: + name: data-kafka-ha-controller-1 + namespace: default + local: + path: /data/controller + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master02 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-controller3 +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: + claimRef: + name: data-kafka-ha-controller-2 + namespace: default + local: + path: /data/controller + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master03 From f0fb71f283c00b118739680026d9ab0dfee71442 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Mon, 22 Sep 2025 16:52:46 +0800 Subject: [PATCH 33/40] =?UTF-8?q?=E5=8F=98=E6=9B=B4kafka=20UI=20SASL?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/ui/deployment.yaml | 17 +++++++++++++++++ charts/kafka-ha/values.yaml | 8 ++++++++ charts/kafka/templates/ui/deployment.yaml | 11 +++++++++++ charts/kafka/values.yaml | 9 +++++++++ 4 files changed, 45 insertions(+) diff --git a/charts/kafka-ha/templates/ui/deployment.yaml b/charts/kafka-ha/templates/ui/deployment.yaml index 152fbc4..a673169 100644 --- a/charts/kafka-ha/templates/ui/deployment.yaml +++ b/charts/kafka-ha/templates/ui/deployment.yaml @@ -52,8 +52,25 @@ spec: - name: KAFKA_CLUSTERS_0_NAME value: {{ include "kafka.fullname" $ }} - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS + {{- if $.Values.broker.auth.enabled }} + {{- /* 使用BROKER监听器端口进行SASL连接 */ -}} + value: {{ printf "%s:%d" (include "broker.headless.serviceAddr" $) ($.Values.broker.containerPort | int) }} + {{- else }} {{- $port := $.Values.broker.containerPort | int }} value: {{ printf "%s:%d" (include "broker.headless.serviceAddr" $) $port }} + {{- end }} + {{- if $.Values.broker.auth.enabled }} + {{- /* SASL认证配置 */ -}} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL + value: "SASL_PLAINTEXT" + - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM + value: {{ $.Values.broker.auth.mechanism | quote }} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG + value: {{ printf "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"%s\" password=\"%s\";" (index $.Values.broker.auth.users 0).username (index $.Values.broker.auth.users 0).password | quote }} + {{- end }} + {{- with .extraEnvs }} + {{- toYaml . | nindent 10 }} + {{- end }} ports: - name: http containerPort: 8080 diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index d032435..5bffde7 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -239,6 +239,12 @@ ui: requests: cpu: 50m memory: 256Mi + ## ui.extraEnvs - 额外的环境变量配置 + extraEnvs: [] + # 示例： + # extraEnvs: + # - name: KAFKA_CLUSTERS_0_READONLY + # value: "false" ## ui.readinessProbe readinessProbe: httpGet: @@ -254,6 +260,8 @@ ui: service: annotations: {} type: ClusterIP + port: 8080 + # nodePort: 30080 # 仅在type为NodePort时使用 ## ui.ingress ingress: enabled: false diff --git a/charts/kafka/templates/ui/deployment.yaml b/charts/kafka/templates/ui/deployment.yaml index 152fbc4..331833b 100644 --- a/charts/kafka/templates/ui/deployment.yaml +++ b/charts/kafka/templates/ui/deployment.yaml @@ -54,6 +54,17 @@ spec: - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS {{- $port := $.Values.broker.containerPort | int }} value: {{ printf "%s:%d" (include "broker.headless.serviceAddr" $) $port }} + {{- if $.Values.broker.auth.enabled }} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL + value: "SASL_PLAINTEXT" + - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM + value: {{ $.Values.broker.auth.mechanism | quote }} + - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG + value: {{ printf "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"%s\" password=\"%s\";" (index $.Values.broker.auth.users 0).username (index $.Values.broker.auth.users 0).password | quote }} + {{- end }} + {{- with .extraEnvs }} + {{- toYaml . | nindent 10 }} + {{- end }} ports: - name: http containerPort: 8080 diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 9782a95..e2b0948 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -259,6 +259,13 @@ ui: requests: cpu: 50m memory: 256Mi + + ## ui.extraEnvs - 额外的环境变量 + extraEnvs: [] + # - name: KAFKA_CLUSTERS_0_READONLY + # value: "false" + # - name: LOGGING_LEVEL_COM_PROVECTUS + # value: "INFO" ## ui.readinessProbe readinessProbe: @@ -276,6 +283,8 @@ ui: service: annotations: {} type: ClusterIP + port: 8080 + # nodePort: 30080 # 仅在type为NodePort时使用 ## ui.ingress ingress: From ea89465ce78bdc7ed7f6a6599774348e077ffb52 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Mon, 22 Sep 2025 17:11:24 +0800 Subject: [PATCH 34/40] =?UTF-8?q?=E5=8F=98=E6=9B=B4kafka=20UI=20SASL?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/ui/deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/kafka-ha/templates/ui/deployment.yaml b/charts/kafka-ha/templates/ui/deployment.yaml index a673169..3fc5ff3 100644 --- a/charts/kafka-ha/templates/ui/deployment.yaml +++ b/charts/kafka-ha/templates/ui/deployment.yaml @@ -53,14 +53,12 @@ spec: value: {{ include "kafka.fullname" $ }} - name: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS {{- if $.Values.broker.auth.enabled }} - {{- /* 使用BROKER监听器端口进行SASL连接 */ -}} value: {{ printf "%s:%d" (include "broker.headless.serviceAddr" $) ($.Values.broker.containerPort | int) }} {{- else }} {{- $port := $.Values.broker.containerPort | int }} value: {{ printf "%s:%d" (include "broker.headless.serviceAddr" $) $port }} {{- end }} {{- if $.Values.broker.auth.enabled }} - {{- /* SASL认证配置 */ -}} - name: KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL value: "SASL_PLAINTEXT" - name: KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM From 4f229f9465702bf75f42584cc4dba9e51ddb9798 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Mon, 22 Sep 2025 17:56:09 +0800 Subject: [PATCH 35/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka=20UI=20base=20aut?= =?UTF-8?q?h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/values.yaml | 8 ++++++++ charts/kafka/values.yaml | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 5bffde7..f21e9dc 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -245,6 +245,14 @@ ui: # extraEnvs: # - name: KAFKA_CLUSTERS_0_READONLY # value: "false" + + # UI认证配置示例 (启用基本用户名密码认证) + # - name: AUTH_TYPE + # value: "LOGIN_FORM" + # - name: SPRING_SECURITY_USER_NAME + # value: "admin" + # - name: SPRING_SECURITY_USER_PASSWORD + # value: "admin123" ## ui.readinessProbe readinessProbe: httpGet: diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index e2b0948..87c8d86 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -261,11 +261,19 @@ ui: memory: 256Mi ## ui.extraEnvs - 额外的环境变量 - extraEnvs: [] + extraEnvs: # - name: KAFKA_CLUSTERS_0_READONLY # value: "false" # - name: LOGGING_LEVEL_COM_PROVECTUS # value: "INFO" + + # UI认证配置示例 (启用基本用户名密码认证) + - name: AUTH_TYPE + value: "LOGIN_FORM" + - name: SPRING_SECURITY_USER_NAME + value: "admin" + - name: SPRING_SECURITY_USER_PASSWORD + value: "admin" ## ui.readinessProbe readinessProbe: From 163c5283fcbdaf8241b7ebf5cd85a926a5eec2bb Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Mon, 22 Sep 2025 21:34:55 +0800 Subject: [PATCH 36/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-exporter=E8=AE=A4?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/values.yaml | 22 +++++++++++++++------- charts/kafka/values.yaml | 19 ++++++++++++++++--- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index f21e9dc..26b6ba1 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -219,6 +219,14 @@ exporter: ## exporter.service service: annotations: {} + + # 额外的命令行参数（用于 SASL 认证等） + extraArgs: + - "--sasl.enabled=true" + - "--sasl.mechanism=SCRAM-SHA-256" + - "--sasl.username=exporter" + - "--sasl.password=Kafka@2025" + ui: enabled: false replicaCount: 1 @@ -240,19 +248,19 @@ ui: cpu: 50m memory: 256Mi ## ui.extraEnvs - 额外的环境变量配置 - extraEnvs: [] + extraEnvs: # 示例： # extraEnvs: # - name: KAFKA_CLUSTERS_0_READONLY # value: "false" # UI认证配置示例 (启用基本用户名密码认证) - # - name: AUTH_TYPE - # value: "LOGIN_FORM" - # - name: SPRING_SECURITY_USER_NAME - # value: "admin" - # - name: SPRING_SECURITY_USER_PASSWORD - # value: "admin123" + - name: AUTH_TYPE + value: "LOGIN_FORM" + - name: SPRING_SECURITY_USER_NAME + value: "admin" + - name: SPRING_SECURITY_USER_PASSWORD + value: "admin" ## ui.readinessProbe readinessProbe: httpGet: diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 87c8d86..64d70d9 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -237,6 +237,19 @@ exporter: ## exporter.service service: annotations: {} + + # 额外的命令行参数（用于 SASL 认证等） + extraArgs: + - "--sasl.enabled=true" + - "--sasl.mechanism=SCRAM-SHA-256" + - "--sasl.username=exporter" + - "--sasl.password=Kafka@2025" + # SASL 认证示例： + # extraArgs: + # - "--sasl.enabled=true" + # - "--sasl.mechanism=SCRAM-SHA-256" + # - "--sasl.username=exporter" + # - "--sasl.password=your-password" ui: enabled: false @@ -268,11 +281,11 @@ ui: # value: "INFO" # UI认证配置示例 (启用基本用户名密码认证) - - name: AUTH_TYPE + - name: AUTH_TYPE value: "LOGIN_FORM" - - name: SPRING_SECURITY_USER_NAME + - name: SPRING_SECURITY_USER_NAME value: "admin" - - name: SPRING_SECURITY_USER_PASSWORD + - name: SPRING_SECURITY_USER_PASSWORD value: "admin" ## ui.readinessProbe From c85cf8071c0a3fbaf0a04ba35632463ef6fe57cd Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Mon, 22 Sep 2025 21:59:42 +0800 Subject: [PATCH 37/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-exporter=E8=AE=A4?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/templates/exporter/deployment.yaml | 7 +++++-- charts/kafka-ha/values.yaml | 6 ++++-- charts/kafka/templates/exporter/deployment.yaml | 5 ++++- charts/kafka/values.yaml | 12 ++++-------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/charts/kafka-ha/templates/exporter/deployment.yaml b/charts/kafka-ha/templates/exporter/deployment.yaml index 6259175..58c4705 100644 --- a/charts/kafka-ha/templates/exporter/deployment.yaml +++ b/charts/kafka-ha/templates/exporter/deployment.yaml @@ -40,9 +40,12 @@ spec: {{- $imageTag := .image.tag | default "latest" }} image: {{ printf "%s:%s" .image.repository $imageTag }} imagePullPolicy: {{ .image.pullPolicy }} - {{- $brokerServicePort := $.Values.broker.service.port | default $.Values.broker.containerPort | int }} + {{- $brokerServicePort := $.Values.broker.containerPort | int }} args: - - {{ printf "--kafka.server=%s:%d" (include "kafka.broker.fullname" $) (int $brokerServicePort) }} + - {{ printf "--kafka.server=%s:%d" (include "broker.headless.serviceAddr" $) (int $brokerServicePort) }} + {{- with .extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} ports: - containerPort: 9308 name: {{ .containerPortName | default "http-metrics"}} diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 26b6ba1..76908e5 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -226,6 +226,8 @@ exporter: - "--sasl.mechanism=SCRAM-SHA-256" - "--sasl.username=exporter" - "--sasl.password=Kafka@2025" + # 如果不需要SASL认证，可以设置为空数组: + # extraArgs: [] ui: enabled: false @@ -258,9 +260,9 @@ ui: - name: AUTH_TYPE value: "LOGIN_FORM" - name: SPRING_SECURITY_USER_NAME - value: "admin" + value: "admin" - name: SPRING_SECURITY_USER_PASSWORD - value: "admin" + value: "admin" ## ui.readinessProbe readinessProbe: httpGet: diff --git a/charts/kafka/templates/exporter/deployment.yaml b/charts/kafka/templates/exporter/deployment.yaml index 6259175..847085b 100644 --- a/charts/kafka/templates/exporter/deployment.yaml +++ b/charts/kafka/templates/exporter/deployment.yaml @@ -40,9 +40,12 @@ spec: {{- $imageTag := .image.tag | default "latest" }} image: {{ printf "%s:%s" .image.repository $imageTag }} imagePullPolicy: {{ .image.pullPolicy }} - {{- $brokerServicePort := $.Values.broker.service.port | default $.Values.broker.containerPort | int }} + {{- $brokerServicePort := $.Values.broker.containerPort | int }} args: - {{ printf "--kafka.server=%s:%d" (include "kafka.broker.fullname" $) (int $brokerServicePort) }} + {{- with .extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} ports: - containerPort: 9308 name: {{ .containerPortName | default "http-metrics"}} diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 64d70d9..0d53a9f 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -244,12 +244,8 @@ exporter: - "--sasl.mechanism=SCRAM-SHA-256" - "--sasl.username=exporter" - "--sasl.password=Kafka@2025" - # SASL 认证示例： - # extraArgs: - # - "--sasl.enabled=true" - # - "--sasl.mechanism=SCRAM-SHA-256" - # - "--sasl.username=exporter" - # - "--sasl.password=your-password" + # 如果不需要SASL认证，可以设置为空数组: + # extraArgs: [] ui: enabled: false @@ -284,9 +280,9 @@ ui: - name: AUTH_TYPE value: "LOGIN_FORM" - name: SPRING_SECURITY_USER_NAME - value: "admin" + value: "admin" - name: SPRING_SECURITY_USER_PASSWORD - value: "admin" + value: "admin" ## ui.readinessProbe readinessProbe: From f20e98e6a8d2f390bf8d123602a97545af55a69f Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Mon, 22 Sep 2025 22:29:42 +0800 Subject: [PATCH 38/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-exporter=E8=AE=A4?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka-ha/values.yaml | 6 +++--- charts/kafka/values.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 76908e5..0ef27ae 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -222,9 +222,9 @@ exporter: # 额外的命令行参数（用于 SASL 认证等） extraArgs: - - "--sasl.enabled=true" - - "--sasl.mechanism=SCRAM-SHA-256" - - "--sasl.username=exporter" + - "--sasl.enabled" + - "--sasl.mechanism=scram-sha256" + - "--sasl.username=kafka" - "--sasl.password=Kafka@2025" # 如果不需要SASL认证，可以设置为空数组: # extraArgs: [] diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 0d53a9f..b1c29ca 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -240,9 +240,9 @@ exporter: # 额外的命令行参数（用于 SASL 认证等） extraArgs: - - "--sasl.enabled=true" - - "--sasl.mechanism=SCRAM-SHA-256" - - "--sasl.username=exporter" + - "--sasl.enabled" + - "--sasl.mechanism=scram-sha256" + - "--sasl.username=kafka" - "--sasl.password=Kafka@2025" # 如果不需要SASL认证，可以设置为空数组: # extraArgs: [] From cfa0e705244a886ef21835867d17371355ccb03e Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Tue, 23 Sep 2025 15:51:53 +0800 Subject: [PATCH 39/40] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka-exporter=E8=AE=A4?= =?UTF-8?q?=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- charts/kafka/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index b1c29ca..025fa88 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -215,7 +215,7 @@ exporter: image: repository: danielqsj/kafka-exporter pullPolicy: IfNotPresent - tag: "latest" + tag: "v1.9.0" podAnnotations: prometheus.io/path: "/metrics" prometheus.io/port: "9308" From c4aa54df472173ec0f83d39a592ec76b447a8144 Mon Sep 17 00:00:00 2001 From: zhouchangrui Date: Wed, 24 Sep 2025 21:43:21 +0800 Subject: [PATCH 40/40] full sasl --- charts/kafka-ha/templates/broker/_helpers.tpl | 17 ++- .../templates/controller/_helpers.tpl | 13 ++ .../templates/controller/cm-jaas.yaml | 23 ++++ .../templates/controller/statefulset.yaml | 10 ++ charts/kafka-ha/values.yaml | 24 ++++ charts/kafka/templates/broker/_helpers.tpl | 19 ++- .../kafka/templates/controller/_helpers.tpl | 13 ++ .../kafka/templates/controller/cm-jaas.yaml | 23 ++++ .../templates/controller/statefulset.yaml | 10 ++ charts/kafka/values.yaml | 26 +++- docs/helm.md | 74 +++++++++++ docs/sasl-scram.md | 122 ++++++++++++++++-- examples/values-production.yml | 2 +- examples/values-sasl-scram.yml | 2 +- 14 files changed, 360 insertions(+), 18 deletions(-) create mode 100644 charts/kafka-ha/templates/controller/cm-jaas.yaml create mode 100644 charts/kafka/templates/controller/cm-jaas.yaml diff --git a/charts/kafka-ha/templates/broker/_helpers.tpl b/charts/kafka-ha/templates/broker/_helpers.tpl index 52c7cf5..f86d396 100644 --- a/charts/kafka-ha/templates/broker/_helpers.tpl +++ b/charts/kafka-ha/templates/broker/_helpers.tpl @@ -153,9 +153,17 @@ broker env value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP {{- if .Values.broker.auth.enabled }} - value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT + {{- if .Values.broker.external.enabled }} + value: CONTROLLER:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT,INTERNAL:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT {{- else }} + value: CONTROLLER:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT,INTERNAL:SASL_PLAINTEXT + {{- end }} + {{- else }} + {{- if .Values.broker.external.enabled }} value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + {{- else }} + value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,INTERNAL:PLAINTEXT + {{- end }} {{- end }} - name: KAFKA_CFG_INTER_BROKER_LISTENER_NAME value: INTERNAL @@ -192,6 +200,13 @@ broker env value: {{ .Values.broker.auth.mechanism | quote }} - name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL value: {{ .Values.broker.auth.mechanism | quote }} +- name: KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL + value: "SASL_PLAINTEXT" +- name: KAFKA_CFG_CONTROLLER_QUORUM_SASL_MECHANISM + value: {{ .Values.broker.auth.mechanism | quote }} +{{- else }} +- name: KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL + value: "PLAINTEXT" {{- end }} {{- if .Values.broker.external.enabled -}} {{- include "broker.externalEnv" $ | nindent 0 }} diff --git a/charts/kafka-ha/templates/controller/_helpers.tpl b/charts/kafka-ha/templates/controller/_helpers.tpl index 624b097..99f2f3c 100644 --- a/charts/kafka-ha/templates/controller/_helpers.tpl +++ b/charts/kafka-ha/templates/controller/_helpers.tpl @@ -17,6 +17,19 @@ controller.containerEnv value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} +{{- if .Values.controller.auth.enabled }} +- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: CONTROLLER:SASL_PLAINTEXT +- name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server_jaas.conf" +- name: KAFKA_CFG_SASL_ENABLED_MECHANISMS + value: {{ .Values.controller.auth.mechanism | quote }} +- name: KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL + value: {{ .Values.controller.auth.mechanism | quote }} +{{- else }} +- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: CONTROLLER:PLAINTEXT +{{- end }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: diff --git a/charts/kafka-ha/templates/controller/cm-jaas.yaml b/charts/kafka-ha/templates/controller/cm-jaas.yaml new file mode 100644 index 0000000..f155f72 --- /dev/null +++ b/charts/kafka-ha/templates/controller/cm-jaas.yaml @@ -0,0 +1,23 @@ +{{- if .Values.controller.auth.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kafka.controller.fullname" . }}-jaas + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller +data: + kafka_server_jaas.conf: | + KafkaServer { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.controller.auth.users 0).username }}" + password="{{ (index .Values.controller.auth.users 0).password }}"; + }; + + Client { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.controller.auth.users 0).username }}" + password="{{ (index .Values.controller.auth.users 0).password }}"; + }; +{{- end }} \ No newline at end of file diff --git a/charts/kafka-ha/templates/controller/statefulset.yaml b/charts/kafka-ha/templates/controller/statefulset.yaml index 35d0e72..b3c96d8 100644 --- a/charts/kafka-ha/templates/controller/statefulset.yaml +++ b/charts/kafka-ha/templates/controller/statefulset.yaml @@ -90,6 +90,10 @@ spec: - mountPath: /entrypoint.sh name: entrypoint-sh subPath: entrypoint.sh + {{- if $.Values.controller.auth.enabled }} + - mountPath: "/etc/kafka/jaas" + name: jaas-config + {{- end }} lifecycle: preStop: exec: @@ -103,6 +107,12 @@ spec: path: entrypoint.sh name: {{ include "kafka.entrypoint.configmapName" $ }} defaultMode: 0744 + {{- if $.Values.controller.auth.enabled }} + - name: jaas-config + configMap: + name: {{ include "kafka.controller.fullname" $ }}-jaas + defaultMode: 0644 + {{- end }} {{- if not .persistence.enabled }} - name: data emptyDir: {} diff --git a/charts/kafka-ha/values.yaml b/charts/kafka-ha/values.yaml index 0ef27ae..8ab9f65 100644 --- a/charts/kafka-ha/values.yaml +++ b/charts/kafka-ha/values.yaml @@ -59,15 +59,26 @@ broker: users: - username: "kafka" password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" + - username: "client" + password: "Client@2025" ## broker.config - server.properties 配置 ## 某些关键配置会被环境变量覆盖，例如: node.id advertised.listeners controller.quorum.voters 等 + ## SASL相关配置将根据auth.enabled动态生成，无需在此处硬编码 config: num.partitions: "6" # default.replication.factor: "1" # min.insync.replicas: "1" default.replication.factor: "3" + # 注意：以下SASL配置已移至模板中动态生成，根据auth.enabled自动配置 + # security.inter.broker.protocol: 将根据auth.enabled动态设置为"SASL_PLAINTEXT"或"PLAINTEXT" + # sasl.mechanism.inter.broker.protocol: 将根据auth.mechanism动态设置 + # controller.quorum.sasl.mechanism: 将根据auth.mechanism动态设置 + # listener.security.protocol.map: 将根据auth.enabled和external.enabled动态设置 + # sasl.enabled.mechanisms: 将根据auth.mechanism动态设置 ## broker.persistence persistence: enabled: true @@ -142,6 +153,19 @@ controller: heapOpts: "-Xms512m -Xmx512m" commonEnvs: [] extraEnvs: [] + + ## SASL SCRAM 认证配置 (与broker共享) + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" + - username: "client" + password: "Client@2025" + ## controller.labels labels: {} ## controller.annotations diff --git a/charts/kafka/templates/broker/_helpers.tpl b/charts/kafka/templates/broker/_helpers.tpl index 62642e9..6e9737d 100644 --- a/charts/kafka/templates/broker/_helpers.tpl +++ b/charts/kafka/templates/broker/_helpers.tpl @@ -156,13 +156,21 @@ broker env value: {{ include "broker.config.advertised.listeners" . }} - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP {{- if .Values.broker.auth.enabled }} - value: CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,PLAINTEXT:PLAINTEXT + {{- if .Values.broker.external.enabled }} + value: CONTROLLER:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,PLAINTEXT:PLAINTEXT {{- else }} + value: CONTROLLER:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT,PLAINTEXT:PLAINTEXT + {{- end }} + {{- else }} + {{- if .Values.broker.external.enabled }} value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT + {{- else }} + value: CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,PLAINTEXT:PLAINTEXT + {{- end }} {{- end }} - name: KAFKA_CFG_INTER_BROKER_LISTENER_NAME {{- if .Values.broker.auth.enabled }} - value: PLAINTEXT + value: BROKER {{- else }} value: BROKER {{- end }} @@ -199,6 +207,13 @@ broker env value: {{ .Values.broker.auth.mechanism | quote }} - name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL value: {{ .Values.broker.auth.mechanism | quote }} +- name: KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL + value: "SASL_PLAINTEXT" +- name: KAFKA_CFG_CONTROLLER_QUORUM_SASL_MECHANISM + value: {{ .Values.broker.auth.mechanism | quote }} +{{- else }} +- name: KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL + value: "PLAINTEXT" {{- end }} {{- if .Values.broker.external.enabled -}} {{- include "broker.externalEnv" $ | nindent 0 }} diff --git a/charts/kafka/templates/controller/_helpers.tpl b/charts/kafka/templates/controller/_helpers.tpl index 624b097..99f2f3c 100644 --- a/charts/kafka/templates/controller/_helpers.tpl +++ b/charts/kafka/templates/controller/_helpers.tpl @@ -17,6 +17,19 @@ controller.containerEnv value: CONTROLLER - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS value: {{ include "kafka.controller.quorum.voters" . }} +{{- if .Values.controller.auth.enabled }} +- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: CONTROLLER:SASL_PLAINTEXT +- name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/etc/kafka/jaas/kafka_server_jaas.conf" +- name: KAFKA_CFG_SASL_ENABLED_MECHANISMS + value: {{ .Values.controller.auth.mechanism | quote }} +- name: KAFKA_CFG_SASL_MECHANISM_CONTROLLER_PROTOCOL + value: {{ .Values.controller.auth.mechanism | quote }} +{{- else }} +- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: CONTROLLER:PLAINTEXT +{{- end }} - name: KAFKA_CLUSTER_ID valueFrom: secretKeyRef: diff --git a/charts/kafka/templates/controller/cm-jaas.yaml b/charts/kafka/templates/controller/cm-jaas.yaml new file mode 100644 index 0000000..f155f72 --- /dev/null +++ b/charts/kafka/templates/controller/cm-jaas.yaml @@ -0,0 +1,23 @@ +{{- if .Values.controller.auth.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kafka.controller.fullname" . }}-jaas + namespace: {{ include "kafka.namespace" $ }} + labels: + {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: controller +data: + kafka_server_jaas.conf: | + KafkaServer { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.controller.auth.users 0).username }}" + password="{{ (index .Values.controller.auth.users 0).password }}"; + }; + + Client { + org.apache.kafka.common.security.scram.ScramLoginModule required + username="{{ (index .Values.controller.auth.users 0).username }}" + password="{{ (index .Values.controller.auth.users 0).password }}"; + }; +{{- end }} \ No newline at end of file diff --git a/charts/kafka/templates/controller/statefulset.yaml b/charts/kafka/templates/controller/statefulset.yaml index 4814943..80a4d49 100644 --- a/charts/kafka/templates/controller/statefulset.yaml +++ b/charts/kafka/templates/controller/statefulset.yaml @@ -90,6 +90,10 @@ spec: - mountPath: /entrypoint.sh name: entrypoint-sh subPath: entrypoint.sh + {{- if $.Values.controller.auth.enabled }} + - mountPath: "/etc/kafka/jaas" + name: jaas-config + {{- end }} lifecycle: preStop: exec: @@ -103,6 +107,12 @@ spec: path: entrypoint.sh name: {{ include "kafka.entrypoint.configmapName" $ }} defaultMode: 0744 + {{- if $.Values.controller.auth.enabled }} + - name: jaas-config + configMap: + name: {{ include "kafka.controller.fullname" $ }}-jaas + defaultMode: 0644 + {{- end }} {{- if not .persistence.enabled }} - name: data emptyDir: {} diff --git a/charts/kafka/values.yaml b/charts/kafka/values.yaml index 025fa88..5c29f76 100644 --- a/charts/kafka/values.yaml +++ b/charts/kafka/values.yaml @@ -65,13 +65,24 @@ broker: users: - username: "kafka" password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" + - username: "client" + password: "Client@2025" ## broker.config - server.properties 配置 ## 某些关键配置会被环境变量覆盖，例如: node.id advertised.listeners controller.quorum.voters 等 + ## SASL相关配置将根据auth.enabled动态生成，无需在此处硬编码 config: num.partitions: "2" # default.replication.factor: "1" # min.insync.replicas: "1" + # 注意：以下SASL配置已移至模板中动态生成，根据auth.enabled自动配置 + # security.inter.broker.protocol: 将根据auth.enabled动态设置为"SASL_PLAINTEXT"或"PLAINTEXT" + # sasl.mechanism.inter.broker.protocol: 将根据auth.mechanism动态设置 + # controller.quorum.sasl.mechanism: 将根据auth.mechanism动态设置 + # listener.security.protocol.map: 将根据auth.enabled和external.enabled动态设置 + # sasl.enabled.mechanisms: 将根据auth.mechanism动态设置 ## broker.persistence persistence: @@ -150,13 +161,26 @@ broker: port: 9095 controller: - enabled: false + enabled: true replicaCount: 1 containerPort: 9091 terminationGracePeriodSeconds: 60 heapOpts: "-Xms512m -Xmx512m" commonEnvs: [] extraEnvs: [] + + ## SASL SCRAM 认证配置 (与broker共享) + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" + - username: "client" + password: "Client@2025" + persistence: enabled: true size: 20Gi diff --git a/docs/helm.md b/docs/helm.md index 6fbb631..9c9920a 100644 --- a/docs/helm.md +++ b/docs/helm.md @@ -98,6 +98,80 @@ broker: > `broker.config` 某些关键配置会被环境变量覆盖，例如: node.id advertised.listeners controller.quorum.voters 等 +## 认证配置 + +### SASL SCRAM 认证 + +#### 启用认证的高可用集群 +```shell +# 部署启用 SASL SCRAM 认证的高可用集群 +helm upgrade --install kafka \ + --namespace kafka-demo \ + --create-namespace \ + --set broker.auth.enabled=true \ + --set broker.auth.mechanism="SCRAM-SHA-256" \ + --set controller.auth.enabled=true \ + --set controller.auth.mechanism="SCRAM-SHA-256" \ + kafka-repo/kafka-ha +``` + +#### 使用配置文件部署 +```shell +# 使用预配置的 SASL SCRAM 配置文件 +helm upgrade --install kafka \ + --namespace kafka-demo \ + --create-namespace \ + -f examples/values-sasl-scram.yml \ + kafka-repo/kafka-ha +``` + +#### 开发环境（无认证） +```shell +# 开发环境部署，关闭认证 +helm upgrade --install kafka \ + --namespace kafka-demo \ + --create-namespace \ + --set broker.auth.enabled=false \ + --set controller.auth.enabled=false \ + kafka-repo/kafka-ha +``` + +### 认证配置说明 + +```yaml +# 启用认证 +broker: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" + +controller: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" +``` + +```yaml +# 关闭认证（开发环境） +broker: + auth: + enabled: false + +controller: + auth: + enabled: false +``` + +> 详细的认证配置请参考 [SASL SCRAM 认证配置指南](sasl-scram.md) + ## 集群外访问 ```yaml diff --git a/docs/sasl-scram.md b/docs/sasl-scram.md index e162fc1..e9b8eb2 100644 --- a/docs/sasl-scram.md +++ b/docs/sasl-scram.md @@ -10,6 +10,7 @@ SASL SCRAM (Salted Challenge Response Authentication Mechanism) 是一种安全 ### 1. 启用 SASL SCRAM 认证 +#### Broker 认证配置 在 `values.yaml` 中添加以下配置： ```yaml @@ -20,27 +21,53 @@ broker: users: - username: "kafka" password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" + - username: "client" + password: "Client@2025" ``` -### 2. 自动配置的参数 +#### Controller 认证配置 +同时需要为 Controller 配置认证： -启用 SASL SCRAM 认证后，以下参数会自动配置： +```yaml +controller: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "Kafka@2025" + - username: "admin" + password: "Admin@2025" +``` + +### 2. 动态配置机制 + +**重要改进**：从 v2.0 开始，SASL 配置采用动态配置机制，根据 `auth.enabled` 自动设置所有相关参数。 -- `sasl.enabled.mechanisms=SCRAM-SHA-256` -- `sasl.mechanism.inter.broker.protocol=SCRAM-SHA-256` -- `security.inter.broker.protocol=SASL_PLAINTEXT` -- `listener.security.protocol.map=CONTROLLER:PLAINTEXT,BROKER:SASL_PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT` +#### 当 `auth.enabled: true` 时，自动配置： +- `KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL=SASL_PLAINTEXT` +- `KAFKA_CFG_SASL_ENABLED_MECHANISMS=SCRAM-SHA-256` +- `KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=SCRAM-SHA-256` +- `KAFKA_CFG_CONTROLLER_QUORUM_SASL_MECHANISM=SCRAM-SHA-256` +- `KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT,...` + +#### 当 `auth.enabled: false` 时，自动配置： +- `KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL=PLAINTEXT` +- `KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,...` +- 不设置 SASL 相关参数 ### 3. 监听器端口配置 Kafka 集群配置了以下监听器： -| 监听器名称 | 端口 | 协议 | 用途 | SASL认证 | -|-----------|------|------|------|----------| -| CONTROLLER | 9093 | PLAINTEXT | Controller间通信 | 否 | -| BROKER | 9092 | SASL_PLAINTEXT | 客户端连接 | 是 | -| INTERNAL | 9094 | PLAINTEXT | Broker间通信 | 否 | -| EXTERNAL | 9095 | SASL_PLAINTEXT | 外部客户端连接 | 是 | +| 监听器名称 | 端口 | 协议 (auth.enabled=true) | 协议 (auth.enabled=false) | 用途 | +|-----------|------|-------------------------|---------------------------|------| +| CONTROLLER | 9093 | SASL_PLAINTEXT | PLAINTEXT | Controller间通信 | +| BROKER | 9092 | SASL_PLAINTEXT | PLAINTEXT | 客户端连接 | +| INTERNAL | 9094 | SASL_PLAINTEXT | PLAINTEXT | Broker间通信 | +| EXTERNAL | 9095 | SASL_PLAINTEXT | PLAINTEXT | 外部客户端连接 | ### 4. Advertised Listeners 配置 @@ -75,6 +102,77 @@ broker: **注意**：确保客户端使用正确的端口和协议进行连接，避免 "no matching listener" 错误。 +## 配置最佳实践 + +### 1. 开发环境 vs 生产环境 + +#### 开发环境配置 +```yaml +# 简化配置，无认证 +broker: + auth: + enabled: false +controller: + auth: + enabled: false +``` + +#### 生产环境配置 +```yaml +# 启用完整认证 +broker: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "StrongPassword123!" + - username: "admin" + password: "AdminPassword456!" +controller: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + users: + - username: "kafka" + password: "StrongPassword123!" +``` + +### 2. 配置迁移指南 + +#### 从旧版本迁移 +如果你使用的是旧版本的配置（在 `broker.config` 中硬编码 SASL 配置），需要进行以下迁移： + +**旧配置（不推荐）**： +```yaml +broker: + config: + security.inter.broker.protocol: "SASL_PLAINTEXT" + sasl.mechanism.inter.broker.protocol: "SCRAM-SHA-256" + # ... 其他硬编码配置 +``` + +**新配置（推荐）**： +```yaml +broker: + auth: + enabled: true + mechanism: "SCRAM-SHA-256" + config: + # 移除所有 SASL 相关的硬编码配置 + # 系统会根据 auth.enabled 自动配置 +``` + +#### 迁移步骤 +1. 备份当前配置 +2. 移除 `broker.config` 中的 SASL 相关配置 +3. 添加 `broker.auth` 和 `controller.auth` 配置 +4. 重新部署并验证 + +### 3. 无认证配置示例 + +参考 文件，了解如何正确配置无认证环境。 + ## 部署步骤 ### 1. 使用示例配置部署 diff --git a/examples/values-production.yml b/examples/values-production.yml index 9874bd8..944b801 100644 --- a/examples/values-production.yml +++ b/examples/values-production.yml @@ -31,4 +31,4 @@ broker: memory: 16Gi requests: cpu: 500m - memory: 6Gi \ No newline at end of file + memory: 6Gi diff --git a/examples/values-sasl-scram.yml b/examples/values-sasl-scram.yml index d7429dd..7ead12e 100644 --- a/examples/values-sasl-scram.yml +++ b/examples/values-sasl-scram.yml @@ -49,4 +49,4 @@ exporter: enabled: false ui: - enabled: false \ No newline at end of file + enabled: false