diff --git a/deployment/helm/README.md b/deployment/helm/README.md index 99e24216f49..fbc08642127 100644 --- a/deployment/helm/README.md +++ b/deployment/helm/README.md @@ -12,15 +12,20 @@ * from source root run the following. This does a very basic test against the web server * ct install --all --helm-extra-set-args="--set=nginx.enabled=false" --debug --config ct.yaml +## Output template to file and inspect +* cd charts/onyx +* helm template test-output . > test-output.yaml + ## Test the entire cluster manually +* cd charts/onyx * helm install onyx . -n onyx --set postgresql.primary.persistence.enabled=false - * the postgres flag is to keep the storage ephemeral for testing, you probably don't want to set that in prod + * the postgres flag is to keep the storage ephemeral for testing. You probably don't want to set that in prod. * no flag for ephemeral vespa storage yet, might be good for testing * kubectl -n onyx port-forward service/onyx-nginx 8080:80 * this will forward the local port 8080 to the installed chart for you to run tests, etc. * When you are finished * helm uninstall onyx -n onyx - * Vespa leaves behind a PVC - delete it if you are completely done + * Vespa leaves behind a PVC. Delete it if you are completely done. * k -n onyx get pvc * k -n onyx delete pvc vespa-storage-da-vespa-0 * If you didn't disable Postgres persistence earlier, you may want to delete that PVC too. \ No newline at end of file diff --git a/deployment/helm/charts/onyx/templates/celery-worker-heavy.yaml b/deployment/helm/charts/onyx/templates/celery-worker-heavy.yaml index 98090fbf81d..3a37186460f 100644 --- a/deployment/helm/charts/onyx/templates/celery-worker-heavy.yaml +++ b/deployment/helm/charts/onyx/templates/celery-worker-heavy.yaml @@ -57,3 +57,25 @@ spec: name: {{ .Values.config.envConfigMapName }} env: {{- include "onyx-stack.envSecrets" . | nindent 12}} + startupProbe: + {{ .Values.celery_shared.startupProbe | toYaml | nindent 12}} + readinessProbe: + {{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe readiness + --filename /tmp/onyx_k8s_heavy_readiness.txt + livenessProbe: + {{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe liveness + --filename /tmp/onyx_k8s_heavy_liveness.txt diff --git a/deployment/helm/charts/onyx/templates/celery-worker-indexing.yaml b/deployment/helm/charts/onyx/templates/celery-worker-indexing.yaml index a9bc2e709e6..83e1fd0144a 100644 --- a/deployment/helm/charts/onyx/templates/celery-worker-indexing.yaml +++ b/deployment/helm/charts/onyx/templates/celery-worker-indexing.yaml @@ -59,3 +59,25 @@ spec: - name: ENABLE_MULTIPASS_INDEXING value: "{{ .Values.celery_worker_indexing.enableMiniChunk }}" {{- include "onyx-stack.envSecrets" . | nindent 12}} + startupProbe: + {{ .Values.celery_shared.startupProbe | toYaml | nindent 12}} + readinessProbe: + {{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe readiness + --filename /tmp/onyx_k8s_indexing_readiness.txt + livenessProbe: + {{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe liveness + --filename /tmp/onyx_k8s_indexing_liveness.txt diff --git a/deployment/helm/charts/onyx/templates/celery-worker-light.yaml b/deployment/helm/charts/onyx/templates/celery-worker-light.yaml index 7c640d829f8..887426a0c28 100644 --- a/deployment/helm/charts/onyx/templates/celery-worker-light.yaml +++ b/deployment/helm/charts/onyx/templates/celery-worker-light.yaml @@ -57,3 +57,25 @@ spec: name: {{ .Values.config.envConfigMapName }} env: {{- include "onyx-stack.envSecrets" . | nindent 12}} + startupProbe: + {{ .Values.celery_shared.startupProbe | toYaml | nindent 12}} + readinessProbe: + {{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe readiness + --filename /tmp/onyx_k8s_light_readiness.txt + livenessProbe: + {{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe liveness + --filename /tmp/onyx_k8s_light_liveness.txt diff --git a/deployment/helm/charts/onyx/templates/celery-worker-monitoring.yaml b/deployment/helm/charts/onyx/templates/celery-worker-monitoring.yaml index c35faf4ce13..d70bc30eb62 100644 --- a/deployment/helm/charts/onyx/templates/celery-worker-monitoring.yaml +++ b/deployment/helm/charts/onyx/templates/celery-worker-monitoring.yaml @@ -57,3 +57,25 @@ spec: name: {{ .Values.config.envConfigMapName }} env: {{- include "onyx-stack.envSecrets" . | nindent 12}} + startupProbe: + {{ .Values.celery_shared.startupProbe | toYaml | nindent 12}} + readinessProbe: + {{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe readiness + --filename /tmp/onyx_k8s_monitoring_readiness.txt + livenessProbe: + {{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe liveness + --filename /tmp/onyx_k8s_monitoring_liveness.txt diff --git a/deployment/helm/charts/onyx/templates/celery-worker-primary.yaml b/deployment/helm/charts/onyx/templates/celery-worker-primary.yaml index 51e763095d4..923ee408a1a 100644 --- a/deployment/helm/charts/onyx/templates/celery-worker-primary.yaml +++ b/deployment/helm/charts/onyx/templates/celery-worker-primary.yaml @@ -57,3 +57,25 @@ spec: name: {{ .Values.config.envConfigMapName }} env: {{- include "onyx-stack.envSecrets" . | nindent 12}} + startupProbe: + {{ .Values.celery_shared.startupProbe | toYaml | nindent 12}} + readinessProbe: + {{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe readiness + --filename /tmp/onyx_k8s_primary_readiness.txt + livenessProbe: + {{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe liveness + --filename /tmp/onyx_k8s_primary_liveness.txt diff --git a/deployment/helm/charts/onyx/templates/celery-worker-user-files-indexing.yaml b/deployment/helm/charts/onyx/templates/celery-worker-user-files-indexing.yaml index 4bddb929d9b..f622dce3be5 100644 --- a/deployment/helm/charts/onyx/templates/celery-worker-user-files-indexing.yaml +++ b/deployment/helm/charts/onyx/templates/celery-worker-user-files-indexing.yaml @@ -57,3 +57,25 @@ spec: name: {{ .Values.config.envConfigMapName }} env: {{- include "onyx-stack.envSecrets" . | nindent 12}} + startupProbe: + {{ .Values.celery_shared.startupProbe | toYaml | nindent 12}} + readinessProbe: + {{ .Values.celery_shared.readinessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe readiness + --filename /tmp/onyx_k8s_userfilesindexing_readiness.txt + livenessProbe: + {{ .Values.celery_shared.livenessProbe | toYaml | nindent 12}} + exec: + command: + - /bin/bash + - -c + - > + python onyx/background/celery/celery_k8s_probe.py + --probe liveness + --filename /tmp/onyx_k8s_userfilesindexing_liveness.txt diff --git a/deployment/helm/charts/onyx/values.yaml b/deployment/helm/charts/onyx/values.yaml index 97d71a3a6ac..f0e68205e4b 100644 --- a/deployment/helm/charts/onyx/values.yaml +++ b/deployment/helm/charts/onyx/values.yaml @@ -359,6 +359,26 @@ celery_shared: repository: onyxdotapp/onyx-backend pullPolicy: IfNotPresent tag: "" # Overrides the image tag whose default is the chart appVersion. + startupProbe: + # startupProbe fails after 2m + exec: + command: ["test", "-f", "/app/onyx/main.py"] + failureThreshold: 24 + periodSeconds: 5 + timeoutSeconds: 3 + readinessProbe: + # readinessProbe fails after 15s + 2m of inactivity + # it's ok to see the readinessProbe fail transiently while the container starts + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 24 + timeoutSeconds: 3 + livenessProbe: + # livenessProbe fails after 5m of inactivity + initialDelaySeconds: 60 + periodSeconds: 60 + failureThreshold: 5 + timeoutSeconds: 3 celery_beat: replicaCount: 1