Skip to content

Commit f6ca266

Browse files
scottcrossenscottcrossen
committed
Expose node locality to pod
1 parent 21ed463 commit f6ca266

10 files changed

+85
-30
lines changed

‎cockroachdb/README.md

+6-2
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,9 @@ For details see the [`values.yaml`](values.yaml) file.
307307
| `conf.locality` | Locality attribute for this deployment | `""` |
308308
| `conf.single-node` | Disable CockroachDB clustering (standalone mode) | `no` |
309309
| `conf.sql-audit-dir` | Directory for SQL audit log | `""` |
310+
| `conf.useHostLocality.enabled` | Use host node labels to set region, zone, host locality keys | `false` |
311+
| `conf.useHostLocality.image.repository` | The init-container image name to query the kubernetes API | `scottcrossen/kube-node-labels` |
312+
| `conf.useHostLocality.image` | The init-container image tag to query the kubernetes API | `1.0.0` |
310313
| `conf.port` | CockroachDB primary serving port in Pods | `26257` |
311314
| `conf.http-port` | CockroachDB HTTP port in Pods | `8080` |
312315
| `conf.path` | CockroachDB data directory mount path | `cockroach-data` |
@@ -352,6 +355,8 @@ For details see the [`values.yaml`](values.yaml) file.
352355
| `service.public.type` | Public Service type | `ClusterIP` |
353356
| `service.public.labels` | Additional labels of public Service | `{"app.kubernetes.io/component": "cockroachdb"}` |
354357
| `service.public.annotations` | Additional annotations of public Service | `{}` |
358+
| `statefulset.serviceAccount.create` | Whether to create a new RBAC service account | `yes` |
359+
| `statefulset.serviceAccount.name` | Name of RBAC service account to use | `""` |
355360
| `service.discovery.labels` | Additional labels of discovery Service | `{"app.kubernetes.io/component": "cockroachdb"}` |
356361
| `service.discovery.annotations` | Additional annotations of discovery Service | `{}` |
357362
| `ingress.enabled` | Enable ingress resource for CockroachDB | `false` |
@@ -375,12 +380,11 @@ For details see the [`values.yaml`](values.yaml) file.
375380
| `init.labels` | Additional labels of init Job and its Pod | `{"app.kubernetes.io/component": "init"}` |
376381
| `init.annotations` | Additional labels of the Pod of init Job | `{}` |
377382
| `init.affinity` | [Affinity rules][2] of init Job Pod | `{}` |
383+
| `init.force` | Create the init pod even if the join URL is set | `false` |
378384
| `init.nodeSelector` | Node labels for init Job Pod assignment | `{}` |
379385
| `init.tolerations` | Node taints to tolerate by init Job Pod | `[]` |
380386
| `init.resources` | Resource requests and limits for the Pod of init Job | `{}` |
381387
| `tls.enabled` | Whether to run securely using TLS certificates | `no` |
382-
| `tls.serviceAccount.create` | Whether to create a new RBAC service account | `yes` |
383-
| `tls.serviceAccount.name` | Name of RBAC service account to use | `""` |
384388
| `tls.certs.provided` | Bring your own certs scenario, i.e certificates are provided | `no` |
385389
| `tls.certs.clientRootSecret` | If certs are provided, secret name for client root cert | `cockroachdb-root` |
386390
| `tls.certs.nodeSecret` | If certs are provided, secret name for node cert | `cockroachdb-node` |

‎cockroachdb/templates/_helpers.tpl

+4-4
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,11 @@ Create chart name and version as used by the chart label.
3333
{{/*
3434
Create the name of the ServiceAccount to use.
3535
*/}}
36-
{{- define "cockroachdb.tls.serviceAccount.name" -}}
37-
{{- if .Values.tls.serviceAccount.create -}}
38-
{{- default (include "cockroachdb.fullname" .) .Values.tls.serviceAccount.name -}}
36+
{{- define "cockroachdb.statefulset.serviceAccount.name" -}}
37+
{{- if .Values.statefulset.serviceAccount.create -}}
38+
{{- default (include "cockroachdb.fullname" .) .Values.statefulset.serviceAccount.name -}}
3939
{{- else -}}
40-
{{- default "default" .Values.tls.serviceAccount.name -}}
40+
{{- default "default" .Values.statefulset.serviceAccount.name -}}
4141
{{- end -}}
4242
{{- end -}}
4343

‎cockroachdb/templates/clusterrole.yaml

+9-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if and .Values.tls.enabled (not .Values.tls.certs.provided) (not .Values.tls.certs.certManager) }}
1+
{{- if or (and .Values.tls.enabled (not .Values.tls.certs.provided) (not .Values.tls.certs.certManager)) .Values.conf.useHostLocality.enabled }}
22
kind: ClusterRole
33
apiVersion: rbac.authorization.k8s.io/v1
44
metadata:
@@ -13,7 +13,14 @@ metadata:
1313
{{- toYaml . | nindent 4 }}
1414
{{- end }}
1515
rules:
16+
{{- if and .Values.tls.enabled (not .Values.tls.certs.provided) (not .Values.tls.certs.certManager) }}
1617
- apiGroups: ["certificates.k8s.io"]
1718
resources: ["certificatesigningrequests"]
1819
verbs: ["create", "get", "watch"]
19-
{{- end }}
20+
{{- end }}
21+
{{- if .Values.conf.useHostLocality.enabled }}
22+
- apiGroups: [""]
23+
resources: ["nodes"]
24+
verbs: ["get"]
25+
{{- end }}
26+
{{- end }}

‎cockroachdb/templates/clusterrolebinding.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if and .Values.tls.enabled (not .Values.tls.certs.provided) (not .Values.tls.certs.certManager) }}
1+
{{- if or (and .Values.tls.enabled (not .Values.tls.certs.provided) (not .Values.tls.certs.certManager)) .Values.conf.useHostLocality.enabled }}
22
kind: ClusterRoleBinding
33
apiVersion: rbac.authorization.k8s.io/v1
44
metadata:
@@ -18,6 +18,6 @@ roleRef:
1818
name: {{ template "cockroachdb.fullname" . }}
1919
subjects:
2020
- kind: ServiceAccount
21-
name: {{ template "cockroachdb.tls.serviceAccount.name" . }}
21+
name: {{ template "cockroachdb.statefulset.serviceAccount.name" . }}
2222
namespace: {{ .Release.Namespace | quote }}
23-
{{- end }}
23+
{{- end }}

‎cockroachdb/templates/job.init.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- if and (eq (len .Values.conf.join) 0) (not (index .Values.conf `single-node`)) }}
1+
{{- if and (or .Values.init.force (eq (len .Values.conf.join) 0)) (not (index .Values.conf `single-node`)) }}
22
{{ template "cockroachdb.tlsValidation" . }}
33
kind: Job
44
apiVersion: batch/v1
@@ -16,9 +16,9 @@ metadata:
1616
{{- with .Values.labels }}
1717
{{- toYaml . | nindent 4 }}
1818
{{- end }}
19-
annotations:
20-
helm.sh/hook: post-install,post-upgrade
21-
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
19+
{{- with .Values.init.jobAnnotations }}
20+
annotations: {{- toYaml . | nindent 4 }}
21+
{{- end }}
2222
spec:
2323
template:
2424
metadata:
@@ -44,7 +44,7 @@ spec:
4444
{{- end }}
4545
{{- end }}
4646
{{- if .Values.tls.enabled }}
47-
serviceAccountName: {{ template "cockroachdb.tls.serviceAccount.name" . }}
47+
serviceAccountName: {{ template "cockroachdb.statefulset.serviceAccount.name" . }}
4848
initContainers:
4949
- name: copy-certs
5050
image: "busybox"

‎cockroachdb/templates/networkpolicy.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
kind: NetworkPolicy
33
apiVersion: {{ template "cockroachdb.networkPolicy.apiVersion" . }}
44
metadata:
5-
name: {{ template "cockroachdb.tls.serviceAccount.name" . }}
5+
name: {{ template "cockroachdb.statefulset.serviceAccount.name" . }}
66
namespace: {{ .Release.Namespace | quote }}
77
labels:
88
helm.sh/chart: {{ template "cockroachdb.chart" . }}

‎cockroachdb/templates/rolebinding.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ roleRef:
1818
name: {{ template "cockroachdb.fullname" . }}
1919
subjects:
2020
- kind: ServiceAccount
21-
name: {{ template "cockroachdb.tls.serviceAccount.name" . }}
21+
name: {{ template "cockroachdb.statefulset.serviceAccount.name" . }}
2222
namespace: {{ .Release.Namespace | quote }}
2323
{{- end }}

‎cockroachdb/templates/serviceaccount.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
{{- if and .Values.tls.enabled .Values.tls.serviceAccount.create }}
1+
{{- if and (or .Values.tls.enabled .Values.conf.useHostLocality.enabled) .Values.statefulset.serviceAccount.create }}
22
kind: ServiceAccount
33
apiVersion: v1
44
metadata:
5-
name: {{ template "cockroachdb.tls.serviceAccount.name" . }}
5+
name: {{ template "cockroachdb.statefulset.serviceAccount.name" . }}
66
namespace: {{ .Release.Namespace | quote }}
77
labels:
88
helm.sh/chart: {{ template "cockroachdb.chart" . }}

‎cockroachdb/templates/statefulset.yaml

+31-4
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,10 @@ spec:
5050
- name: {{ template "cockroachdb.fullname" . }}.self-signed-certs.registry
5151
{{- end }}
5252
{{- end }}
53+
{{- if or .Values.tls.enabled .Values.conf.useHostLocality.enabled }}
54+
serviceAccountName: {{ template "cockroachdb.statefulset.serviceAccount.name" . }}
55+
{{- end }}
5356
{{- if .Values.tls.enabled }}
54-
serviceAccountName: {{ template "cockroachdb.tls.serviceAccount.name" . }}
55-
{{- if .Values.tls.enabled }}
5657
initContainers:
5758
- name: copy-certs
5859
image: "busybox"
@@ -71,7 +72,21 @@ spec:
7172
mountPath: /cockroach-certs/
7273
- name: certs-secret
7374
mountPath: /certs/
74-
{{- end }}
75+
{{- end }}
76+
{{- if .Values.conf.useHostLocality.enabled }}
77+
- name: get-node-info
78+
image: {{ .Values.conf.useHostLocality.image.repository }}:{{ .Values.conf.useHostLocality.image.tag }}
79+
imagePullPolicy: IfNotPresent
80+
env:
81+
- name: NODE
82+
valueFrom:
83+
fieldRef:
84+
fieldPath: spec.nodeName
85+
- name: OUTPUT_DIR
86+
value: /output
87+
volumeMounts:
88+
- name: node-data
89+
mountPath: /output
7590
{{- end }}
7691
{{- if or .Values.statefulset.nodeAffinity .Values.statefulset.podAffinity .Values.statefulset.podAntiAffinity }}
7792
affinity:
@@ -202,8 +217,12 @@ spec:
202217
--max-offset={{ . }}
203218
{{- end }}
204219
--max-sql-memory={{ index .Values.conf `max-sql-memory` }}
205-
{{- with .Values.conf.locality }}
220+
{{- if .Values.conf.useHostLocality.enabled }}
221+
--locality=$(/node-data/topology.sh){{ with .Values.conf.locality }},{{ . }}{{ end }}
222+
{{- else }}
223+
{{- with .Values.conf.locality }}
206224
--locality={{ . }}
225+
{{- end }}
207226
{{- end }}
208227
{{- with index .Values.conf `sql-audit-dir` }}
209228
--sql-audit-dir={{ . }}
@@ -257,6 +276,10 @@ spec:
257276
mountPath: /cockroach/log-config
258277
readOnly: true
259278
{{- end }}
279+
{{- if .Values.conf.useHostLocality.enabled }}
280+
- name: node-data
281+
mountPath: /node-data
282+
{{- end }}
260283
livenessProbe:
261284
{{- if .Values.statefulset.customLivenessProbe }}
262285
{{ toYaml .Values.statefulset.customLivenessProbe | nindent 12 }}
@@ -339,6 +362,10 @@ spec:
339362
secret:
340363
secretName: {{ template "cockroachdb.fullname" . }}-log-config
341364
{{- end }}
365+
{{- if .Values.conf.useHostLocality.enabled }}
366+
- name: node-data
367+
emptyDir: {}
368+
{{- end }}
342369
{{- if .Values.storage.persistentVolume.enabled }}
343370
volumeClaimTemplates:
344371
- metadata:

‎cockroachdb/values.yaml

+23-6
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,15 @@ conf:
122122
# locality: planet=earth,province=manitoba,colo=secondary,power=3
123123
locality: ""
124124

125+
# Queries the kubernetes API to get the region, zone, and host of the
126+
# instance a given pod is running on and adds that to the front of the
127+
# locality key set
128+
useHostLocality:
129+
enabled: false
130+
image:
131+
repository: scottcrossen/kube-node-labels
132+
tag: 1.1.0
133+
125134
# Run CockroachDB instances in standalone mode with replication disabled
126135
# (replication factor = 1).
127136
# Enabling this option makes the following values to be ignored:
@@ -261,6 +270,13 @@ statefulset:
261270
# initialDelaySeconds: 30
262271
# periodSeconds: 5
263272

273+
serviceAccount:
274+
# Specifies whether this ServiceAccount should be created.
275+
create: true
276+
# The name of this ServiceAccount to use.
277+
# If not set and `create` is `true`, then a name is auto-generated.
278+
name: ""
279+
264280
service:
265281
ports:
266282
# You can set a different external and internal gRPC ports and their name.
@@ -359,6 +375,11 @@ init:
359375
labels:
360376
app.kubernetes.io/component: init
361377

378+
# Annotations to add to the init job
379+
jobAnnotations:
380+
helm.sh/hook: post-install,post-upgrade
381+
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
382+
362383
# Additional annotations to apply to the Pod of this Job.
363384
annotations: {}
364385

@@ -387,16 +408,12 @@ init:
387408
# cpu: "10m"
388409
# memory: "128Mi"
389410

411+
# Create the init job even if this has the join URL set
412+
force: false
390413

391414
# Whether to run securely using TLS certificates.
392415
tls:
393416
enabled: true
394-
serviceAccount:
395-
# Specifies whether this ServiceAccount should be created.
396-
create: true
397-
# The name of this ServiceAccount to use.
398-
# If not set and `create` is `true`, then a name is auto-generated.
399-
name: ""
400417
certs:
401418
# Bring your own certs scenario. If provided, tls.init section will be ignored.
402419
provided: false

0 commit comments

Comments
 (0)