feat: add health check; fix: rabbitmq credentials, minio ci set up

Author: vanessavmac

.envs/.ci/.postgres

@@ -3,3 +3,4 @@ POSTGRES_PORT=5432
 POSTGRES_DB=ami-ci
 POSTGRES_USER=4JXkOnTAeDmDyIapSRrGEE
 POSTGRES_PASSWORD=d4xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI
+DATABASE_URL=postgres://xekSryPnqczJXkOnTAeDmDyIapSRrGEE:iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI@postgres:5432/ami

.envs/.local/.django

@@ -12,9 +12,6 @@ DJANGO_SUPERUSER_PASSWORD=localadmin
 # Redis
 REDIS_URL=redis://redis:6379/0
 
-# RabbitMQ
-CELERY_BROKER_URL=amqp://user:password@rabbitmq:5672//
-
 # Celery / Flower
 CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT
 CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf

.envs/.local/.postgres

@@ -3,3 +3,4 @@ POSTGRES_PORT=5432
 POSTGRES_DB=ami
 POSTGRES_USER=xekSryPnqczJXkOnTAeDmDyIapSRrGEE
 POSTGRES_PASSWORD=iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI
+DATABASE_URL=postgres://xekSryPnqczJXkOnTAeDmDyIapSRrGEE:iMRQjJEGflj5xojpnJU3OzPQ0apSCLP1oHR1TYvyMzAlF5KpE9HFL6MPlnbDibwI@postgres:5432/ami

ami/ml/models/pipeline.py

@@ -334,6 +334,7 @@ def handle_async_process_images(
         )
 
         task_id = str(uuid.uuid4())
+        # use transaction on commit to ensure source images and other project details are finished saving
         transaction.on_commit(
             lambda: process_pipeline_request.apply_async(
                 args=[prediction_request.dict(), project_id],

ami/ml/signals.py

@@ -54,12 +54,7 @@ def subscribe_celeryworker_to_pipeline_queues(sender, **kwargs) -> bool:
     if not pipelines:
         # TODO: kinda hacky. is there a way to unify the django and celery logs
         # to more easily see which queues the worker is subscribed to?
-        raise ValueError(
-            "No pipelines found; cannot subscribe to any queues. "
-            "If the database was just reset and migrated, this error might be expected. "
-            "Check both django and celery logs to ensure worker is subscribed to the project queues. "
-            "Alternatively, restart the celery worker again."
-        )
+        raise ValueError("No pipelines found; cannot subscribe to any queues.")
 
     for slug in pipelines:
         queue_name = f"ml-pipeline-{slug}"
@@ -81,13 +76,11 @@ def pipeline_created(sender, instance, created, **kwargs):
         queue_name = f"ml-pipeline-{instance.slug}"
         worker_name = get_worker_name()
 
-        if not worker_name:
-            logger.warning(
-                "Could not determine worker name; cannot subscribe to new queue "
-                f"{queue_name}. This might be an expected error if the worker hasn't "
-                "started or is ready to accept connections."
-            )
-            return
+        assert worker_name, (
+            "Could not determine worker name; cannot subscribe to new queue "
+            f"{queue_name}. This might be an expected error if the worker hasn't "
+            "started or is ready to accept connections."
+        )
 
         celery_app.control.add_consumer(queue_name, destination=[worker_name])
         logger.info(f"Queue '{queue_name}' successfully added to worker '{worker_name}'")

compose/local/django/Dockerfile

@@ -62,10 +62,6 @@ RUN sed -i 's/\r$//g' /start
 RUN chmod +x /start
 
 
-COPY ./compose/local/django/celery/worker/start /start-celeryworker
-RUN sed -i 's/\r$//g' /start-celeryworker
-RUN chmod +x /start-celeryworker
-
 COPY ./compose/local/django/celery/beat/start /start-celerybeat
 RUN sed -i 's/\r$//g' /start-celerybeat
 RUN chmod +x /start-celerybeat

docker-compose.ci.yml

@@ -4,6 +4,8 @@ name: antenna-ci
 volumes:
   ami_ci_postgres_data:
     driver: local
+  minio_ci_data:
+    driver: local
 
 services:
   django: &django
@@ -12,12 +14,19 @@ services:
       dockerfile: ./compose/local/django/Dockerfile
     volumes:
       - .:/app:z
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     depends_on:
-      - postgres
-      - redis
-      - minio-init
-      - ml_backend
-      - celeryworker
+      postgres:
+        condition: service_started
+      redis:
+        condition: service_started
+      minio-init:
+        condition: service_started
+      rabbitmq:
+        condition: service_started
+      celeryworker:  # required to subscribe the worker to the pipelines in the db
+        condition: service_healthy
     env_file:
       - ./.envs/.ci/.django
       - ./.envs/.ci/.postgres
@@ -38,11 +47,21 @@ services:
   celeryworker:
     <<: *django
     depends_on:
-      - postgres
       - rabbitmq
-    volumes:
-      - ./compose/local/django/celery/worker/start:/start-celeryworker
-    command: /start-celeryworker
+    # start the worker with antenna_celeryworker to ensure it's discoverable by ami.ml.signals.get_worker_name
+    command:
+      - sh
+      - -c
+      - |
+        python manage.py migrate &&
+        python -m celery -A config.celery_app worker --queues=antenna -n antenna_celeryworker@%h -l INFO
+    healthcheck:
+      # make sure DATABASE_URL is inside the ./.envs/.ci/.postgres
+      test: ["CMD-SHELL", "celery -A config.celery_app inspect ping -d antenna_celeryworker@$(hostname) | grep -q pong"]
+      interval: 10s
+      timeout: 50s
+      retries: 5
+      start_period: 10s
 
   rabbitmq:
     image: rabbitmq:3-management
@@ -52,6 +71,8 @@ services:
   minio:
     image: minio/minio:RELEASE.2024-11-07T00-52-20Z
     command: minio server --console-address ":9001" /data
+    volumes:
+      - "minio_ci_data:/data"
     env_file:
       - ./.envs/.ci/.django
     healthcheck:
@@ -65,7 +86,8 @@ services:
     env_file:
       - ./.envs/.ci/.django
     depends_on:
-      - minio
+      minio:
+        condition: service_healthy
     volumes:
       - ./compose/local/minio/init.sh:/etc/minio/init.sh
     entrypoint: /etc/minio/init.sh

docker-compose.yml

@@ -20,11 +20,16 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
     depends_on:
-      - postgres
-      - redis
-      - minio-init
-      - rabbitmq
-      - celeryworker # required to subscribe the worker to the pipelines in the db
+      postgres:
+        condition: service_started
+      redis:
+        condition: service_started
+      minio-init:
+        condition: service_started
+      rabbitmq:
+        condition: service_started
+      celeryworker:  # required to subscribe the worker to the pipelines in the db
+        condition: service_healthy
     volumes:
       - .:/app:z
     env_file:
@@ -99,11 +104,22 @@ services:
     # Also make sure to install debugpy in your requirements/local.txt
     ports:
       - "5678:5678"
-    command: python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -n antenna_celeryworker@%h  -l INFO
-    volumes:
-      - ./compose/local/django/celery/worker/start:/start-celeryworker
+    # start the worker with antenna_celeryworker to ensure it's discoverable by ami.ml.signals.get_worker_name
+    command:
+      - sh
+      - -c
+      - |
+        python manage.py migrate &&
+        python -m debugpy --listen 0.0.0.0:5678 -m celery -A config.celery_app worker --queues=antenna -n antenna_celeryworker@%h -l INFO
     depends_on:
       - rabbitmq
+    healthcheck:
+      # make sure DATABASE_URL is inside the ./.envs/.local/.postgres
+      test: ["CMD-SHELL", "celery -A config.celery_app inspect ping -d antenna_celeryworker@$(hostname) | grep -q pong"]
+      interval: 10s
+      timeout: 50s
+      retries: 5
+      start_period: 10s
 
   celerybeat:
     <<: *django
@@ -126,8 +142,8 @@ services:
       - "5672:5672"
       - "15672:15672"
     environment:
-      RABBITMQ_DEFAULT_USER: user
-      RABBITMQ_DEFAULT_PASS: password
+      RABBITMQ_DEFAULT_USER: rabbituser
+      RABBITMQ_DEFAULT_PASS: rabbitpass
     networks:
       - antenna_network
 
@@ -162,8 +178,10 @@ services:
     env_file:
       - ./.envs/.local/.django
     depends_on:
-      - minio
-      - minio-proxy
+      minio:
+        condition: service_healthy
+      minio-proxy:
+        condition: service_started
     volumes:
       - ./compose/local/minio/init.sh:/etc/minio/init.sh
     entrypoint: /etc/minio/init.sh
@@ -187,7 +205,7 @@ services:
       context: ./processing_services/minimal
     command: ./celery_worker/start_celery.sh
     environment:
-      - CELERY_BROKER_URL=amqp://user:password@rabbitmq:5672//
+      - CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672//
     extra_hosts:
       - minio:host-gateway
     networks:

processing_services/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       context: ./minimal
     command: ./celery_worker/start_celery.sh
     environment:
-      - CELERY_BROKER_URL=amqp://user:password@rabbitmq:5672//
+      - CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672//
     extra_hosts:
       - minio:host-gateway
     networks:
@@ -42,7 +42,7 @@ services:
       context: ./example
     command: ./celery_worker/start_celery.sh
     environment:
-      - CELERY_BROKER_URL=amqp://user:password@rabbitmq:5672//
+      - CELERY_BROKER_URL=amqp://rabbituser:rabbitpass@rabbitmq:5672//
     extra_hosts:
       - minio:host-gateway
     networks: