Merge branch 'main' into feat/quickstart-auto-process

Author: mihow

.envs/.local/.django

@@ -37,3 +37,8 @@ MINIO_DEFAULT_BUCKET=ami
 MINIO_STORAGE_USE_HTTPS=False
 MINIO_TEST_BUCKET=ami-test
 MINIO_BROWSER_REDIRECT_URL=http://minio:9001
+
+# Default processing service (local)
+DEFAULT_PROCESSING_SERVICE_NAME=Local Processing Service
+DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000
+# DEFAULT_PIPELINES_ENABLED=random,constant  # When set to None, all pipelines will be enabled.

.envs/.production/.django-example

@@ -53,3 +53,8 @@ DJANGO_ACCOUNT_ALLOW_REGISTRATION=True
 # Gunicorn
 # ------------------------------------------------------------------------------
 WEB_CONCURRENCY=4
+
+# Default processing service
+DEFAULT_PROCESSING_SERVICE_NAME="AMI Data Companion"
+DEFAULT_PROCESSING_SERVICE_ENDPOINT=https://ml.antenna.insectai.org/
+DEFAULT_PIPELINES_ENABLED=global_moths_2024,quebec_vermont_moths_2023,panama_moths_2023,uk_denmark_moths_2023

ami/main/models.py

@@ -15,7 +15,7 @@
 from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.core.files.storage import default_storage
-from django.db import IntegrityError, models
+from django.db import IntegrityError, models, transaction
 from django.db.models import Q
 from django.db.models.fields.files import ImageFieldFile
 from django.db.models.signals import pre_delete
@@ -34,6 +34,7 @@
 
 if typing.TYPE_CHECKING:
     from ami.jobs.models import Job
+    from ami.ml.models import ProcessingService
 
 logger = logging.getLogger(__name__)
 
@@ -89,20 +90,57 @@ def get_media_url(path: str) -> str:
 as_choices = lambda x: [(i, i) for i in x]  # noqa: E731
 
 
-def create_default_device(project: "Project") -> "Device":
+def get_or_create_default_device(project: "Project") -> "Device":
     """Create a default device for a project."""
     device, _created = Device.objects.get_or_create(name="Default device", project=project)
     logger.info(f"Created default device for project {project}")
     return device
 
 
-def create_default_research_site(project: "Project") -> "Site":
+def get_or_create_default_research_site(project: "Project") -> "Site":
     """Create a default research site for a project."""
     site, _created = Site.objects.get_or_create(name="Default site", project=project)
     logger.info(f"Created default research site for project {project}")
     return site
 
 
+def get_or_create_default_deployment(
+    project: "Project", site: "Site | None" = None, device: "Device | None" = None
+) -> "Deployment":
+    """Create a default deployment for a project."""
+    deployment, _created = Deployment.objects.get_or_create(
+        name="Default Station",
+        project=project,
+        research_site=site,
+        device=device,
+    )
+    logger.info(f"Created default deployment for project {project}")
+    return deployment
+
+
+def get_or_create_default_collection(project: "Project") -> "SourceImageCollection":
+    """Create a default collection for a project for all images, updated dynamically."""
+    collection, _created = SourceImageCollection.objects.get_or_create(
+        name="All Images",
+        project=project,
+    )
+    logger.info(f"Created default collection for project {project}")
+    return collection
+
+
+def get_or_create_default_project(user: User) -> "Project":
+    """
+    Create a default project for a user.
+
+    Default related objects like devices and research sites will be created
+    when the project is saved for the first time.
+    If the project already exists, it will be returned without modification.
+    """
+    project, _created = Project.objects.get_or_create(name="Scratch Project", owner=user, create_defaults=True)
+    logger.info(f"Created default project for user {user}")
+    return project
+
+
 class ProjectQuerySet(models.QuerySet):
     def filter_by_user(self, user: User):
         """
@@ -115,6 +153,39 @@ class ProjectManager(models.Manager):
     def get_queryset(self) -> ProjectQuerySet:
         return ProjectQuerySet(self.model, using=self._db)
 
+    def create(self, create_defaults: bool = True, **kwargs) -> "Project":
+        """
+        Create a new Project and related models with defaults.
+
+        Args:
+            create_defaults: Whether to create default related models
+            **kwargs: Model field values
+
+        Returns:
+            Created Project instance
+        """
+        with transaction.atomic():
+            project_instance = super().create(**kwargs)
+            logger.info(f"Created project: {project_instance.name}")
+
+            if create_defaults:
+                self.create_related_defaults(project_instance)
+
+            return project_instance
+
+    def create_related_defaults(self, project: "Project"):
+        """Create default device, and other related models for this project if they don't exist."""
+        device = get_or_create_default_device(project=project)
+        site = get_or_create_default_research_site(project=project)
+        if not project.deployments.exists():
+            get_or_create_default_deployment(project=project, site=site, device=device)
+        if not project.sourceimage_collections.exists():
+            get_or_create_default_collection(project=project)
+        if not project.processing_services.exists():
+            from ami.ml.models.processing_service import get_or_create_default_processing_service
+
+            get_or_create_default_processing_service(project=project)
+
 
 @final
 class Project(BaseModel):
@@ -140,6 +211,8 @@ class Project(BaseModel):
     devices: models.QuerySet["Device"]
     sites: models.QuerySet["Site"]
     jobs: models.QuerySet["Job"]
+    sourceimage_collections: models.QuerySet["SourceImageCollection"]
+    processing_services: models.QuerySet["ProcessingService"]
 
     objects = ProjectManager()
 
@@ -177,21 +250,10 @@ def summary_data(self):
 
         return plots
 
-    def create_related_defaults(self):
-        """Create default device, and other related models for this project if they don't exist."""
-        if not self.devices.exists():
-            create_default_device(project=self)
-        if not self.sites.exists():
-            create_default_research_site(project=self)
-
     def save(self, *args, **kwargs):
-        new_project = bool(self._state.adding)
         super().save(*args, **kwargs)
         # Add owner to members
         self.ensure_owner_membership()
-        if new_project:
-            logger.info(f"Created new project {self}")
-            self.create_related_defaults()
 
     class Permissions:
         """CRUD Permission names follow the convention: `create_<model>`, `update_<model>`,
@@ -1192,6 +1254,7 @@ class S3StorageSource(BaseModel):
     # last_check_duration = models.DurationField(null=True, blank=True)
     # use_signed_urls = models.BooleanField(default=False)
     project = models.ForeignKey(Project, on_delete=models.SET_NULL, null=True, related_name="storage_sources")
+    # @TODO allow multiple projects to share the same S3StorageSource
 
     deployments: models.QuerySet["Deployment"]
 

ami/main/tests.py

@@ -4,7 +4,7 @@
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import connection, models
-from django.test import TestCase
+from django.test import TestCase, override_settings
 from guardian.shortcuts import get_perms
 from PIL import Image
 from rest_framework import status
@@ -29,6 +29,7 @@
     group_images_into_events,
 )
 from ami.ml.models.pipeline import Pipeline
+from ami.ml.models.project_pipeline_config import ProjectPipelineConfig
 from ami.tests.fixtures.main import create_captures, create_occurrences, create_taxa, setup_test_project
 from ami.tests.fixtures.storage import populate_bucket
 from ami.users.models import User
@@ -37,6 +38,170 @@
 logger = logging.getLogger(__name__)
 
 
+class TestProjectSetup(TestCase):
+    def test_project_creation(self):
+        project = Project.objects.create(name="New Project with Defaults", create_defaults=True)
+        self.assertIsInstance(project, Project)
+
+    def test_default_related_models(self):
+        """Test that the default related models are created correctly when a project is created."""
+        project = Project.objects.create(name="New Project with Defaults", create_defaults=True)
+
+        # Check that the project has a default deployment
+        self.assertGreaterEqual(project.deployments.count(), 1)
+        deployment = project.deployments.first()
+        self.assertIsInstance(deployment, Deployment)
+
+        # Check that the deployment has a default site
+        self.assertGreaterEqual(project.sites.count(), 1)
+        site = project.sites.first()
+        self.assertIsInstance(site, Site)
+
+        # Check that the deployment has a default device
+        self.assertGreaterEqual(project.devices.count(), 1)
+        device = project.devices.first()
+        self.assertIsInstance(device, Device)
+
+        # Check that the project has a default source image collection
+        self.assertGreaterEqual(project.sourceimage_collections.count(), 1)
+        collection = project.sourceimage_collections.first()
+        self.assertIsInstance(collection, SourceImageCollection)
+
+    # Disable this test for now, as it requires a more complex setup
+    def no_test_default_permissions(self):
+        pass
+
+    @override_settings(
+        DEFAULT_PROCESSING_SERVICE_NAME="Default Processing Service",
+        DEFAULT_PROCESSING_SERVICE_ENDPOINT="http://ml_backend:2009/",
+    )
+    def test_processing_service_if_configured(self):
+        """
+        Test that the default processing service is created if the environment variables are set.
+        """
+        from ami.ml.models.processing_service import get_or_create_default_processing_service
+
+        project = Project.objects.create(name="Test Project for Processing Service", create_defaults=False)
+
+        service = get_or_create_default_processing_service(project=project, register_pipelines=False)
+        self.assertIsNotNone(service, "Default processing service should be created if environment variables are set.")
+        assert service is not None  # For type checking
+        self.assertIsNotNone(service.endpoint_url)
+        self.assertIsNotNone(service.name)
+        self.assertGreaterEqual(project.processing_services.count(), 1)
+
+    @override_settings(
+        DEFAULT_PROCESSING_SERVICE_NAME=None,
+        DEFAULT_PROCESSING_SERVICE_ENDPOINT=None,
+    )
+    def test_processing_service_if_not_configured(self):
+        """
+        Test that the default processing service is not created if the environment variables are not set.
+        """
+        from ami.ml.models.processing_service import get_or_create_default_processing_service
+
+        project = Project.objects.create(name="Test Project for Processing Service", create_defaults=False)
+
+        service = get_or_create_default_processing_service(project=project)
+        self.assertIsNone(
+            service, "Default processing service should not be created if environment variables are not set."
+        )
+
+    @override_settings(
+        DEFAULT_PROCESSING_SERVICE_NAME="Default Processing Service",
+        DEFAULT_PROCESSING_SERVICE_ENDPOINT="http://ml_backend:2000/",
+        DEFAULT_PIPELINES_ENABLED=[],  # All pipelines DISABLED by default
+    )
+    def test_processing_service_with_disabled_pipelines(self):
+        """
+        Test that the default processing service is created with all pipelines disabled
+        if DEFAULT_PIPELINES_ENABLED is any empty list.
+        """
+        project = Project.objects.create(name="Test Project for Processing Service", create_defaults=True)
+        processing_service = project.processing_services.first()
+        assert processing_service is not None
+        # There should be at least two pipelines created by default
+        self.assertGreaterEqual(processing_service.pipelines.count(), 2)
+        # All pipelines should be disabled by default
+        project_pipeline_configs = ProjectPipelineConfig.objects.filter(project=project)
+        for config in project_pipeline_configs:
+            self.assertFalse(
+                config.enabled,
+                f"Pipeline {config.pipeline.name} should be disabled for project {project.name}.",
+            )
+
+    @override_settings(
+        DEFAULT_PROCESSING_SERVICE_NAME="Default Processing Service",
+        DEFAULT_PROCESSING_SERVICE_ENDPOINT="http://ml_backend:2000/",
+        DEFAULT_PIPELINES_ENABLED=None,  # All pipelines ENABLED by default
+    )
+    def test_processing_service_with_enabled_pipelines(self):
+        """
+        Test that the default processing service is created with all pipelines enabled
+        if the DEFAULT_PIPELINES_ENABLED setting is None (or missing).
+        """
+        project = Project.objects.create(name="Test Project for Processing Service", create_defaults=True)
+        processing_service = project.processing_services.first()
+        assert processing_service is not None
+        # There should be at least two pipelines created by default
+        self.assertGreaterEqual(processing_service.pipelines.count(), 2)
+        # All pipelines should be enabled by default
+        project_pipeline_configs = ProjectPipelineConfig.objects.filter(project=project)
+        for config in project_pipeline_configs:
+            self.assertTrue(
+                config.enabled,
+                f"Pipeline {config.pipeline.name} should be enabled for project {project.name}.",
+            )
+
+    @override_settings(
+        DEFAULT_PROCESSING_SERVICE_NAME="Default Processing Service",
+        DEFAULT_PROCESSING_SERVICE_ENDPOINT="http://ml_backend:2000/",  # should have at least two pipelines
+        DEFAULT_PIPELINES_ENABLED=["constant"],
+    )
+    def test_existing_processing_service_new_project(self):
+        """
+        Create a new project, enable all pipelines.
+        Create a 2nd project, ensure that the same processing service is used and only the enabled pipelines are
+        registered.
+        """
+        enabled_pipelines = ["constant"]
+
+        project_one = Project.objects.create(name="Test Project One", create_defaults=True)
+
+        # Enable all pipelines for the first project
+        ProjectPipelineConfig.objects.filter(project=project_one).update(enabled=True)
+
+        project_two = Project.objects.create(name="Test Project Two", create_defaults=True)
+
+        project_one_processing_service = project_one.processing_services.first()
+        project_two_processing_service = project_two.processing_services.first()
+
+        assert project_one_processing_service is not None
+        assert project_two_processing_service is not None
+
+        # Ensure only the same processing service instance is used (and they are not None)
+        self.assertEqual(
+            project_one_processing_service,
+            project_two_processing_service,
+            "Both projects should use the same processing service instance.",
+        )
+
+        # Ensure that only the enabled pipelines are enabled for the second project
+        project_two_pipeline_configs = ProjectPipelineConfig.objects.filter(project=project_two)
+        self.assertGreaterEqual(project_two_pipeline_configs.count(), 2, "Project should have at least two pipelines.")
+        for config in project_two_pipeline_configs:
+            if config.pipeline.slug in enabled_pipelines:
+                self.assertTrue(
+                    config.enabled,
+                    f"Pipeline {config.pipeline.name} should be enabled for project {project_two.name}.",
+                )
+            else:
+                self.assertFalse(
+                    config.enabled,
+                    f"Pipeline {config.pipeline.name} should not be enabled for project {project_two.name}.",
+                )
+
+
 class TestImageGrouping(TestCase):
     def setUp(self) -> None:
         print(f"Currently active database: {connection.settings_dict}")