Skip to content

Commit ede673e

Browse files
Subash-MohanSubash-Mohan
committed
Add celery worker for user file processing with autoscaling configuration and readiness/liveness probes in Helm chart
1 parent ec39a81 commit ede673e

File tree

3 files changed

+163
-0
lines changed

3 files changed

+163
-0
lines changed

‎deployment/helm/charts/onyx/templates/celery-worker-user-file-processing-scaledobject.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{{- if .Values.celery_worker_user_file_processing.autoscaling.enabled }}
2+
apiVersion: keda.sh/v1alpha1
3+
kind: ScaledObject
4+
metadata:
5+
name: {{ include "onyx-stack.fullname" . }}-celery-worker-user-file-processing
6+
labels:
7+
{{- include "onyx-stack.labels" . | nindent 4 }}
8+
spec:
9+
scaleTargetRef:
10+
apiVersion: apps/v1
11+
kind: Deployment
12+
name: {{ include "onyx-stack.fullname" . }}-celery-worker-user-file-processing
13+
minReplicaCount: {{ .Values.celery_worker_user_file_processing.autoscaling.minReplicas | default 1 }}
14+
maxReplicaCount: {{ .Values.celery_worker_user_file_processing.autoscaling.maxReplicas | default 10 }}
15+
pollingInterval: {{ .Values.celery_worker_user_file_processing.autoscaling.pollingInterval | default 30 }}
16+
cooldownPeriod: {{ .Values.celery_worker_user_file_processing.autoscaling.cooldownPeriod | default 300 }}
17+
idleReplicaCount: {{ .Values.celery_worker_user_file_processing.autoscaling.idleReplicaCount | default 1 }}
18+
fallback:
19+
failureThreshold: {{ .Values.celery_worker_user_file_processing.autoscaling.failureThreshold | default 3 }}
20+
replicas: {{ .Values.celery_worker_user_file_processing.autoscaling.fallbackReplicas | default 1 }}
21+
triggers:
22+
{{- if .Values.celery_worker_user_file_processing.autoscaling.targetCPUUtilizationPercentage }}
23+
- type: cpu
24+
metricType: Utilization
25+
metadata:
26+
value: "{{ .Values.celery_worker_user_file_processing.autoscaling.targetCPUUtilizationPercentage }}"
27+
{{- end }}
28+
{{- if .Values.celery_worker_user_file_processing.autoscaling.targetMemoryUtilizationPercentage }}
29+
- type: memory
30+
metricType: Utilization
31+
metadata:
32+
value: "{{ .Values.celery_worker_user_file_processing.autoscaling.targetMemoryUtilizationPercentage }}"
33+
{{- end }}
34+
{{- if .Values.celery_worker_user_file_processing.autoscaling.customTriggers }}
35+
{{- toYaml .Values.celery_worker_user_file_processing.autoscaling.customTriggers | nindent 4 }}
36+
{{- end }}
37+
{{- end }}
38+
39+

‎deployment/helm/charts/onyx/templates/celery-worker-user-file-processing.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{{- if gt (int .Values.celery_worker_user_file_processing.replicaCount) 0 }}
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
name: {{ include "onyx-stack.fullname" . }}-celery-worker-user-file-processing
6+
labels:
7+
{{- include "onyx-stack.labels" . | nindent 4 }}
8+
spec:
9+
{{- if not .Values.celery_worker_user_file_processing.autoscaling.enabled }}
10+
replicas: {{ .Values.celery_worker_user_file_processing.replicaCount }}
11+
{{- end }}
12+
selector:
13+
matchLabels:
14+
{{- include "onyx-stack.selectorLabels" . | nindent 6 }}
15+
{{- if .Values.celery_worker_user_file_processing.deploymentLabels }}
16+
{{- toYaml .Values.celery_worker_user_file_processing.deploymentLabels | nindent 6 }}
17+
{{- end }}
18+
template:
19+
metadata:
20+
annotations:
21+
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
22+
{{- with .Values.celery_worker_user_file_processing.podAnnotations }}
23+
{{- toYaml . | nindent 8 }}
24+
{{- end }}
25+
labels:
26+
{{- include "onyx-stack.labels" . | nindent 8 }}
27+
{{- with .Values.celery_worker_user_file_processing.podLabels }}
28+
{{- toYaml . | nindent 8 }}
29+
{{- end }}
30+
spec:
31+
{{- with .Values.imagePullSecrets }}
32+
imagePullSecrets:
33+
{{- toYaml . | nindent 8 }}
34+
{{- end }}
35+
serviceAccountName: {{ include "onyx-stack.serviceAccountName" . }}
36+
securityContext:
37+
{{- toYaml .Values.celery_shared.podSecurityContext | nindent 8 }}
38+
{{- with .Values.celery_worker_user_file_processing.nodeSelector }}
39+
nodeSelector:
40+
{{- toYaml . | nindent 8 }}
41+
{{- end }}
42+
containers:
43+
- name: celery-worker-user-file-processing
44+
securityContext:
45+
{{- toYaml .Values.celery_shared.securityContext | nindent 12 }}
46+
image: "{{ .Values.celery_shared.image.repository }}:{{ .Values.celery_shared.image.tag | default .Values.global.version }}"
47+
imagePullPolicy: {{ .Values.global.pullPolicy }}
48+
command:
49+
[
50+
"celery",
51+
"-A",
52+
"onyx.background.celery.versioned_apps.user_file_processing",
53+
"worker",
54+
"--loglevel=INFO",
55+
"--hostname=user-file-processing@%n",
56+
"-Q",
57+
"user_file_processing,user_file_project_sync",
58+
]
59+
resources:
60+
{{- toYaml .Values.celery_worker_user_file_processing.resources | nindent 12 }}
61+
envFrom:
62+
- configMapRef:
63+
name: {{ .Values.config.envConfigMapName }}
64+
env:
65+
{{- include "onyx-stack.envSecrets" . | nindent 12}}
66+
{{- with .Values.celery_worker_user_file_processing.volumeMounts }}
67+
volumeMounts:
68+
{{- toYaml . | nindent 12 }}
69+
{{- end }}
70+
startupProbe:
71+
{{ .Values.celery_shared.startupProbe | toYaml | nindent 12}}
72+
readinessProbe:
73+
{{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}}
74+
livenessProbe:
75+
{{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}}
76+
{{- with .Values.celery_worker_user_file_processing.volumes }}
77+
volumes:
78+
{{- toYaml . | nindent 8 }}
79+
{{- end }}
80+
{{- end }}
81+
82+

‎deployment/helm/charts/onyx/values.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,12 +346,18 @@ celery_shared:
346346
readinessProbe:
347347
# readinessProbe fails after 15s + 2m of inactivity
348348
# it's ok to see the readinessProbe fail transiently while the container starts
349+
#NOTE(subash): I can't able to run the pod in the local without this line - so added it here. (justin)Please verify if it is required.
350+
exec:
351+
command: ["test", "-f", "/app/onyx/main.py"]
349352
initialDelaySeconds: 15
350353
periodSeconds: 5
351354
failureThreshold: 24
352355
timeoutSeconds: 3
353356
livenessProbe:
354357
# livenessProbe fails after 5m of inactivity
358+
#NOTE(subash): I can't able to run the pod in the local without this line - so added it here. (justin)Please verify if it is required.
359+
exec:
360+
command: ["test", "-f", "/app/onyx/main.py"]
355361
initialDelaySeconds: 60
356362
periodSeconds: 60
357363
failureThreshold: 5
@@ -592,6 +598,41 @@ celery_worker_user_files_indexing:
592598
tolerations: []
593599
affinity: {}
594600

601+
celery_worker_user_file_processing:
602+
replicaCount: 1
603+
autoscaling:
604+
enabled: false
605+
minReplicas: 1
606+
maxReplicas: 10
607+
targetCPUUtilizationPercentage: 80
608+
targetMemoryUtilizationPercentage: 80
609+
# KEDA specific configurations
610+
pollingInterval: 30 # seconds
611+
cooldownPeriod: 300 # seconds
612+
idleReplicaCount: 1 # minimum replicas when idle
613+
failureThreshold: 3 # number of failures before fallback
614+
fallbackReplicas: 1 # replicas to maintain on failure
615+
# Custom triggers for advanced KEDA configurations
616+
customTriggers: []
617+
podAnnotations: {}
618+
podLabels:
619+
scope: onyx-backend-celery
620+
app: celery-worker-user-file-processing
621+
deploymentLabels:
622+
app: celery-worker-user-file-processing
623+
resources:
624+
requests:
625+
cpu: 2000m
626+
memory: 6Gi
627+
limits:
628+
cpu: 4000m
629+
memory: 12Gi
630+
volumes: [] # Additional volumes on the output Deployment definition.
631+
volumeMounts: [] # Additional volumeMounts on the output Deployment definition.
632+
nodeSelector: {}
633+
tolerations: []
634+
affinity: {}
635+
595636
slackbot:
596637
enabled: true
597638
replicaCount: 1
@@ -852,6 +893,7 @@ configMap:
852893
CELERY_WORKER_DOCPROCESSING_CONCURRENCY: ""
853894
CELERY_WORKER_LIGHT_CONCURRENCY: ""
854895
CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER: ""
896+
CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY: ""
855897
# OnyxBot SlackBot Configs
856898
ONYX_BOT_DISABLE_DOCS_ONLY_ANSWER: ""
857899
ONYX_BOT_DISPLAY_ERROR_MSGS: ""

0 commit comments

Comments
 (0)