Improve initial setup for local dev

.envs/.local/.django

@@ -1,24 +1,28 @@
 # General
-# ------------------------------------------------------------------------------
 DJANGO_SETTINGS_MODULE="config.settings.local"
 USE_DOCKER=yes
 DJANGO_DEBUG=True
 IPYTHONDIR=/app/.ipython
+
+# Default superuser for local development
+DJANGO_SUPERUSER_EMAIL=antenna@insectai.org
+DJANGO_SUPERUSER_PASSWORD=localadmin
+
 # Redis
-# ------------------------------------------------------------------------------
 REDIS_URL=redis://redis:6379/0
 
-# Celery
-# ------------------------------------------------------------------------------
-
-# Flower
+# Celery / Flower
 CELERY_FLOWER_USER=QSocnxapfMvzLqJXSsXtnEZqRkBtsmKT
 CELERY_FLOWER_PASSWORD=BEQgmCtgyrFieKNoGTsux9YIye0I7P5Q7vEgfJD2C4jxmtHDetFaE2jhS7K7rxaf
 
 # This is the hostname for the frontend server
-EXTERNAL_HOSTNAME=localhost:3000
+EXTERNAL_HOSTNAME=localhost:4000
 DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost:3000,http://localhost:4000
 
+# CORS
+CORS_ALLOWED_ORIGINS=http://localhost:3000,http://localhost:4000
+
+# Minio local S3 storage backend
 MINIO_ENDPOINT=http://minio:9000
 MINIO_ROOT_USER=amistorage
 MINIO_ROOT_PASSWORD=amistorage

README.md

@@ -6,14 +6,40 @@ Platform for processing and reviewing images from automated insect monitoring st
 
 ## Quick Start
 
-The platform uses Docker Compose to run all services locally for development. Install Docker Desktop and run the following command:
+Antenna uses [Docker](https://docs.docker.com/get-docker/) & [Docker Compose](https://docs.docker.com/compose/install/) to run all services locally for development.
 
-    $ docker compose up
+1) Install Docker for your host operating (Linux, macOS, Windows)
 
-- Web UI: http://localhost:4000
-- API Browser: http://localhost:8000/api/v2/
+2) Add the following to your `/etc/hosts` file in order to see and process the demo source images. This makes the hostname `minio` and alias for `localhost` so the same image URLs can be viewed in the host machine's web browser and be processed by the ML services. This can be skipped if you are using an external image storage service.
+
+```
+    127.0.0.1 minio
+```
+
+2) The following commands will build all services, run them in the background, and then stream the logs.
+
+```sh
+    docker compose up -d
+    docker compose logs -f django celeryworker ui
+    # Ctrl+c to close the logs
+```
+
+3) Access the platform the following URLs:
+
+- Primary web interface: http://localhost:4000
+- API browser: http://localhost:8000/api/v2/
 - Django admin: http://localhost:8000/admin/
-- OpenAPI / Swagger Docs: http://localhost:8000/api/v2/docs/
+- OpenAPI / Swagger documentation: http://localhost:8000/api/v2/docs/
+
+A default user will be created with the following credentials. Use these to log into the web UI or the Django admin.
+
+- Email: `antenna@insectai.org`
+- Password: `localadmin`
+
+4) Stop all services with:
+
+    $ docker compose down
+
 
 ## Development
 
@@ -90,6 +116,12 @@ pip install -r requirements/local.txt
 
     docker compose run --rm django python manage.py createsuperuser
 
+##### Create a fresh demo project with synthetic data
+
+```bash
+docker compose run --rm django python manage.py create_demo_project
+```
+
 ##### Run tests
 
 ```bash
@@ -108,6 +140,12 @@ docker compose run --rm django python manage.py test -k pattern
 docker compose run --rm django python manage.py test -k pattern --failfast --pdb
 ```
 
+##### Run management scripts
+
+```bash
+docker compose run django python manage.py --help
+```
+
 ##### Launch the Django shell:
 
     docker-compose exec django python manage.py shell
@@ -154,14 +192,15 @@ To configure a project connect to the Minio service, you can use the following c
 Endpoint URL: http://minio:9000
 Access key: amistorage
 Secret access key: amistorage
-Public base URL: http://localhost:9000/ami/
+Public base URL: http://minio:9000/ami/
 Bucket: ami
 ```
 
 - Open the Minio web interface at http://localhost:9001 and login with the access key and secret access key.
 - Upload some test images to a subfolder in the `ami` bucket (one subfolder per deployment)
 - Give the bucket or folder anonymous access using the "Anonymous access" button in the Minio web interface.
-- You _can_ test private buckets and presigned URLs, but you will need to add an entry to your local /etc/hosts file to map the `minio` hostname to localhost.
+- Both public and private buckets with presigned URLs should work.
+- Add an entry to your local `/etc/hosts` file to map the `minio` hostname to localhost so the same image URLs can be viewed in your host machine's browser and processed in the backend containers.
 
 ```
 127.0.0.1 minio

ami/jobs/tests.py

@@ -105,7 +105,7 @@ def test_create_job_unauthenticated(self):
         }
         self.client.force_authenticate(user=None)
         resp = self.client.post(jobs_create_url, job_data)
-        self.assertEqual(resp.status_code, 403)
+        self.assertEqual(resp.status_code, 401)
 
     def _create_job(self, name: str, start_now: bool = True):
         jobs_create_url = reverse_with_params("api:job-list")
@@ -173,7 +173,7 @@ def test_run_job_unauthenticated(self):
         jobs_run_url = reverse_with_params("api:job-run", args=[self.job.pk])
         self.client.force_authenticate(user=None)
         resp = self.client.post(jobs_run_url)
-        self.assertEqual(resp.status_code, 403)
+        self.assertEqual(resp.status_code, 401)
 
     def test_cancel_job(self):
         # This cannot be tested until we have a way to cancel jobs

ami/main/api/serializers.py

@@ -1339,7 +1339,7 @@ class StorageSourceSerializer(DefaultSerializer):
     # endpoint_url = serializers.URLField(required=False, allow_blank=True)
     # @TODO the endpoint needs to support host names without a TLD extension like "minio:9000"
     endpoint_url = serializers.CharField(required=False, allow_blank=True, allow_null=True)
-    public_base_url = serializers.URLField(required=False, allow_blank=True, allow_null=True)
+    public_base_url = serializers.CharField(required=False, allow_blank=True, allow_null=True)
 
     class Meta:
         model = S3StorageSource

ami/main/apps.py

@@ -8,6 +8,6 @@ class MainConfig(AppConfig):
     verbose_name = _("Main")
 
     def ready(self):
-        from ami.tests.fixtures.signals import setup_complete_test_project
+        from ami.tests.fixtures.signals import initialize_demo_project
 
-        post_migrate.connect(setup_complete_test_project, sender=self)
+        post_migrate.connect(initialize_demo_project, sender=self)

ami/main/management/commands/create_demo_project.py

@@ -0,0 +1,45 @@
+import time
+
+from django.core.management.base import BaseCommand
+
+from ami.main.models import Deployment, Detection, Device, Event, Occurrence, Project, SourceImage, TaxaList, Taxon
+from ami.ml.models import Algorithm, Pipeline
+from ami.tests.fixtures.main import create_complete_test_project, create_local_admin_user
+
+
+class Command(BaseCommand):
+    r"""Create example data needed for development and tests."""
+
+    help = "Create example data needed for development and tests"
+
+    def add_arguments(self, parser):
+        # Add option to delete existing data
+        parser.add_argument(
+            "--delete",
+            action="store_true",
+            help="Delete existing data before creating new demo project",
+        )
+
+    def handle(self, *args, **options):
+        if options["delete"]:
+            self.stdout.write(self.style.WARNING("! Deleting existing data !"))
+            time.sleep(2)
+            for model in [
+                Project,
+                Device,
+                Deployment,
+                TaxaList,
+                Taxon,
+                Event,
+                SourceImage,
+                Detection,
+                Occurrence,
+                Algorithm,
+                Pipeline,
+            ]:
+                self.stdout.write(f"Deleting all {model._meta.verbose_name_plural} and related objects")
+                model.objects.all().delete()
+
+        self.stdout.write("Creating test project")
+        create_complete_test_project()
+        create_local_admin_user()

ami/ml/models/pipeline.py

@@ -174,13 +174,14 @@ def process_images(
 
     resp = requests.post(endpoint_url, json=request_data.dict())
     if not resp.ok:
+        try:
+            msg = resp.json()["detail"]
+        except Exception:
+            msg = resp.content
         if job:
-            try:
-                msg = resp.json()["detail"]
-            except Exception:
-                msg = resp.content
-
             job.logger.error(msg)
+        else:
+            logger.error(msg)
 
         resp.raise_for_status()
 

ami/tests/fixtures/main.py

@@ -1,9 +1,12 @@
 import datetime
 import logging
+import os
 import pathlib
 import random
 import uuid
 
+from django.db import transaction
+
 from ami.main.models import (
     Deployment,
     Detection,
@@ -348,3 +351,26 @@ def create_occurrences(
         assert occurrence.best_prediction is not None
         assert occurrence.determination is not None
         assert occurrence.determination_score is not None
+
+
+def create_complete_test_project():
+    with transaction.atomic():
+        project, deployment = setup_test_project(reuse=False)
+        frame_data = create_captures_from_files(deployment)
+        taxa_list = create_taxa(project)
+        create_occurrences_from_frame_data(frame_data, taxa_list=taxa_list)
+        logger.info(f"Created test project {project}")
+
+
+def create_local_admin_user():
+    from django.core.management import call_command
+
+    logger.info("Creating superuser with the credentials set in environment variables")
+    try:
+        call_command("createsuperuser", interactive=False)
+    except Exception as e:
+        logger.error(f"Failed to create superuser: {e}")
+
+    email = os.environ.get("DJANGO_SUPERUSER_EMAIL", "Unknown")
+    password = os.environ.get("DJANGO_SUPERUSER_PASSWORD", "Unknown")
+    logger.info(f"Test user credentials: {email} / {password}")

ami/tests/fixtures/signals.py

@@ -1,31 +1,19 @@
 import logging
 
 from django.conf import settings
-from django.db import transaction
 
 from ami.main.models import Project
 
-from .main import (
-    create_captures_from_files,
-    create_occurrences_from_frame_data,
-    create_taxa,
-    setup_test_project,
-    update_site_settings,
-)
+from .main import create_complete_test_project, create_local_admin_user, update_site_settings
 
 logger = logging.getLogger(__name__)
 
 
-# Signal receiver function
-def setup_complete_test_project(sender, **kwargs):
-    # Test if any project exists or if force is set
-    if Project.objects.exists() and not kwargs.get("force", False):
-        return
-
-    with transaction.atomic():
+def initialize_demo_project(sender, **kwargs):
+    """
+    Signal handler to create a demo project after `migrate` is run.
+    """
+    if not Project.objects.exists():
         update_site_settings(domain=settings.EXTERNAL_HOSTNAME)
-        project, deployment = setup_test_project(reuse=False)
-        frame_data = create_captures_from_files(deployment)
-        taxa_list = create_taxa(project)
-        create_occurrences_from_frame_data(frame_data, taxa_list=taxa_list)
-        logger.info(f"Created test project {project}")
+        create_complete_test_project()
+        create_local_admin_user()

ami/tests/fixtures/storage.py

@@ -16,7 +16,7 @@
     secret_access_key=settings.S3_TEST_SECRET,
     bucket_name=settings.S3_TEST_BUCKET,
     prefix="test_prefix",
-    # public_base_url="http://localhost:9000/test",
+    public_base_url=f"http://minio:9000/{settings.S3_TEST_BUCKET}/test_prefix",
     # public_base_url="http://minio:9001",
 )
 

ami/tests/test_storage.py

@@ -147,12 +147,12 @@ def test_presigned_url(self):
 class TestS3PrefixUtils(TestCase):
     def setUp(self):
         self.config = s3.S3Config(
-            endpoint_url="http://localhost:9000",
+            endpoint_url="http://minio:9000",
             access_key_id="minioadmin",
             secret_access_key="minioadmin",
             bucket_name="test_bucket",
             prefix="test_prefix",
-            public_base_url="http://localhost:9000/test",
+            public_base_url="http://minio:9000/test",
         )
 
     def test_key_with_prefix_no_subdir(self):