Merge branch 'main' into feat/taxa-tags

Author: mihow

ami/main/api/views.py

@@ -646,6 +646,7 @@ class SourceImageCollectionViewSet(DefaultViewSet, ProjectMixin):
     ]
     filterset_fields = ["method"]
     ordering_fields = [
+        "id",
         "created_at",
         "updated_at",
         "name",

ami/main/migrations/0060_alter_sourceimagecollection_method.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.10 on 2025-07-08 16:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main", "0059_alter_project_options"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="sourceimagecollection",
+            name="method",
+            field=models.CharField(
+                choices=[
+                    ("common_combined", "common_combined"),
+                    ("random", "random"),
+                    ("stratified_random", "stratified_random"),
+                    ("interval", "interval"),
+                    ("manual", "manual"),
+                    ("starred", "starred"),
+                    ("random_from_each_event", "random_from_each_event"),
+                    ("last_and_random_from_each_event", "last_and_random_from_each_event"),
+                    ("greatest_file_size_from_each_event", "greatest_file_size_from_each_event"),
+                    ("detections_only", "detections_only"),
+                    ("full", "full"),
+                ],
+                default="common_combined",
+                max_length=255,
+            ),
+        ),
+    ]

ami/main/models.py

@@ -612,6 +612,9 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J
             job.progress.add_stage("Update deployment cache")
             job.update_progress()
 
+        # Regroup source images  if needed
+        if deployment_event_needs_update(deployment):
+            group_images_into_events(deployment)
         self.save()
         self.update_calculated_fields(save=True)
 
@@ -1093,6 +1096,23 @@ def group_images_into_events(
     return events
 
 
+def deployment_event_needs_update(deployment: Deployment) -> bool:
+    """
+    Returns True if there are any SourceImages in the deployment
+    that haven't been assigned to an `Event`.
+
+    Note: This does not detect if images were deleted from the deployment
+    after being grouped. We currently have limited support for image deletion,
+    so handling that is out of scope for this check.
+    """
+
+    ungrouped_images_exist = SourceImage.objects.filter(deployment=deployment, event__isnull=True).exists()
+
+    logger.debug(f"Deployment {deployment.pk}: ungrouped images exist = {ungrouped_images_exist}")
+
+    return ungrouped_images_exist
+
+
 def delete_empty_events(deployment: Deployment, dry_run=False):
     """
     Delete events that have no images, occurrences or other related records.
@@ -3022,7 +3042,7 @@ def html(self) -> str:
 
 
 _SOURCE_IMAGE_SAMPLING_METHODS = [
-    "common_combined",
+    "common_combined",  # Deprecated
     "random",
     "stratified_random",
     "interval",
@@ -3032,6 +3052,7 @@ def html(self) -> str:
     "last_and_random_from_each_event",
     "greatest_file_size_from_each_event",
     "detections_only",
+    "full",
 ]
 
 
@@ -3157,7 +3178,16 @@ def taxa_count(self) -> int | None:
         # This should always be pre-populated using queryset annotations
         return None
 
-    def get_queryset(self):
+    def get_queryset(
+        self,
+        hour_start: int | None = None,
+        hour_end: int | None = None,
+        month_start: int | None = None,
+        month_end: int | None = None,
+        date_start: str | None = None,
+        date_end: str | None = None,
+        deployment_ids: list[int] | None = None,
+    ):
         return SourceImage.objects.filter(project=self.project)
 
     @classmethod
@@ -3184,33 +3214,17 @@ def populate_sample(self, job: "Job | None" = None):
             self.save()
             task_logger.info(f"Done sampling and saving captures to {self}")
 
-    def sample_random(self, size: int = 100):
-        """Create a random sample of source images"""
-
-        qs = self.get_queryset()
-        return qs.order_by("?")[:size]
-
-    def sample_manual(self, image_ids: list[int]):
-        """Create a sample of source images based on a list of source image IDs"""
-
-        qs = self.get_queryset()
-        return qs.filter(id__in=image_ids)
-
-    def sample_common_combined(
+    def _filter_sample(
         self,
-        minute_interval: int | None = None,
-        max_num: int | None = None,
-        shuffle: bool = True,  # This is applicable if max_num is set and minute_interval is not set
+        qs: models.QuerySet,
         hour_start: int | None = None,
         hour_end: int | None = None,
         month_start: int | None = None,
         month_end: int | None = None,
         date_start: str | None = None,
         date_end: str | None = None,
         deployment_ids: list[int] | None = None,
-    ) -> models.QuerySet | typing.Generator[SourceImage, None, None]:
-        qs = self.get_queryset()
-
+    ):
         if deployment_ids is not None:
             qs = qs.filter(deployment__in=deployment_ids)
         if date_start is not None:
@@ -3235,6 +3249,66 @@ def sample_common_combined(
         elif hour_end is not None:
             qs = qs.filter(timestamp__hour__lte=hour_end)
 
+        return qs
+
+    def sample_random(
+        self,
+        size: int = 100,
+        hour_start: int | None = None,
+        hour_end: int | None = None,
+        month_start: int | None = None,
+        month_end: int | None = None,
+        date_start: str | None = None,
+        date_end: str | None = None,
+        deployment_ids: list[int] | None = None,
+    ):
+        """Create a random sample of source images"""
+
+        qs = self.get_queryset()
+        qs = self._filter_sample(
+            qs=qs,
+            hour_start=hour_start,
+            hour_end=hour_end,
+            month_start=month_start,
+            month_end=month_end,
+            date_start=date_start,
+            date_end=date_end,
+            deployment_ids=deployment_ids,
+        )
+        return qs.order_by("?")[:size]
+
+    def sample_manual(self, image_ids: list[int]):
+        """Create a sample of source images based on a list of source image IDs"""
+
+        qs = self.get_queryset()
+        return qs.filter(id__in=image_ids)
+
+    # Deprecated
+    def sample_common_combined(
+        self,
+        minute_interval: int | None = None,
+        max_num: int | None = None,
+        shuffle: bool = True,  # This is applicable if max_num is set and minute_interval is not set
+        hour_start: int | None = None,
+        hour_end: int | None = None,
+        month_start: int | None = None,
+        month_end: int | None = None,
+        date_start: str | None = None,
+        date_end: str | None = None,
+        deployment_ids: list[int] | None = None,
+    ) -> models.QuerySet | typing.Generator[SourceImage, None, None]:
+        qs = self.get_queryset()
+        qs = self._filter_sample(
+            qs=qs,
+            hour_start=hour_start,
+            hour_end=hour_end,
+            month_start=month_start,
+            month_end=month_end,
+            date_start=date_start,
+            date_end=date_end,
+            deployment_ids=deployment_ids,
+        )
+
         if minute_interval is not None:
             # @TODO can this be done in the database and return a queryset?
             # this currently returns a list of source images
@@ -3250,11 +3324,31 @@ def sample_common_combined(
         return qs
 
     def sample_interval(
-        self, minute_interval: int = 10, exclude_events: list[int] = [], deployment_id: int | None = None
+        self,
+        minute_interval: int = 10,
+        exclude_events: list[int] = [],
+        deployment_id: int | None = None,  # Deprecated
+        hour_start: int | None = None,
+        hour_end: int | None = None,
+        month_start: int | None = None,
+        month_end: int | None = None,
+        date_start: str | None = None,
+        date_end: str | None = None,
+        deployment_ids: list[int] | None = None,
     ):
         """Create a sample of source images based on a time interval"""
 
         qs = self.get_queryset()
+        qs = self._filter_sample(
+            qs=qs,
+            hour_start=hour_start,
+            hour_end=hour_end,
+            month_start=month_start,
+            month_end=month_end,
+            date_start=date_start,
+            date_end=date_end,
+            deployment_ids=deployment_ids,
+        )
         if deployment_id:
             qs = qs.filter(deployment=deployment_id)
         if exclude_events:
@@ -3314,6 +3408,31 @@ def sample_detections_only(self):
         qs = self.get_queryset()
         return qs.filter(detections__isnull=False).distinct()
 
+    def sample_full(
+        self,
+        hour_start: int | None = None,
+        hour_end: int | None = None,
+        month_start: int | None = None,
+        month_end: int | None = None,
+        date_start: str | None = None,
+        date_end: str | None = None,
+        deployment_ids: list[int] | None = None,
+    ):
+        """Sample all source images"""
+
+        qs = self.get_queryset()
+        qs = self._filter_sample(
+            qs=qs,
+            hour_start=hour_start,
+            hour_end=hour_end,
+            month_start=month_start,
+            month_end=month_end,
+            date_start=date_start,
+            date_end=date_end,
+            deployment_ids=deployment_ids,
+        )
+        return qs.all().distinct()
+
     @classmethod
     def get_or_create_starred_collection(cls, project: Project) -> "SourceImageCollection":
         """

ami/main/tests.py

@@ -30,6 +30,7 @@
 )
 from ami.ml.models.pipeline import Pipeline
 from ami.tests.fixtures.main import create_captures, create_occurrences, create_taxa, setup_test_project
+from ami.tests.fixtures.storage import populate_bucket
 from ami.users.models import User
 from ami.users.roles import BasicMember, Identifier, ProjectManager
 
@@ -1572,3 +1573,56 @@ def test_project_manager_permissions_(self):
         self._test_sourceimageupload_permissions(
             user=self.project_manager, permission_map=self.PERMISSIONS_MAPS["project_manager"]["sourceimageupload"]
         )
+
+
+class TestDeploymentSyncCreatesEvents(TestCase):
+    def test_sync_creates_events_and_updates_counts(self):
+        # Set up a new project and deployment with test data
+        project, deployment = setup_test_project(reuse=False)
+
+        # Populate the object store with image data
+        assert deployment.data_source is not None
+        populate_bucket(
+            config=deployment.data_source.config,
+            subdir=f"deployment_{deployment.pk}",
+            skip_existing=False,
+        )
+
+        # Sync captures
+        deployment.sync_captures()
+
+        # Refresh and check results
+        deployment.refresh_from_db()
+        initial_events = Event.objects.filter(deployment=deployment)
+        initial_events_count = initial_events.count()
+
+        # Assertions
+        self.assertTrue(initial_events.exists(), "Expected events to be created")
+        self.assertEqual(
+            deployment.events_count, initial_events.count(), "Deployment events_count should match actual events"
+        )
+        # Simulate new images added to object store
+        populate_bucket(
+            config=deployment.data_source.config,
+            subdir=f"deployment_{deployment.pk}",
+            skip_existing=False,
+            num_nights=2,
+            images_per_day=5,
+            minutes_interval=120,
+        )
+
+        # Sync again
+        deployment.sync_captures()
+        deployment.refresh_from_db()
+        updated_events = Event.objects.filter(deployment=deployment)
+
+        # Assertions for second sync
+        self.assertGreater(
+            updated_events.count(), initial_events_count, "New events should be created after adding new images"
+        )
+        self.assertEqual(
+            deployment.events_count,
+            updated_events.count(),
+            "Deployment events_count should reflect updated event count",
+        )
+        logger.info(f"Initial events count: {initial_events_count}, Updated events count: {updated_events.count()}")