Merge branch 'main' into fix/fix-duplicate-check

Author: mihow

ami/main/admin.py

@@ -9,6 +9,7 @@
 
 import ami.utils
 from ami import tasks
+from ami.ml.tasks import remove_duplicate_classifications
 
 from .models import (
     BlogPost,
@@ -57,6 +58,16 @@ class ProjectAdmin(admin.ModelAdmin[Project]):
 
     list_display = ("name", "priority", "active", "created_at", "updated_at")
 
+    @admin.action(description="Remove duplicate classifications from all detections")
+    def _remove_duplicate_classifications(self, request: HttpRequest, queryset: QuerySet[Project]) -> None:
+        task_ids = []
+        for project in queryset:
+            task = remove_duplicate_classifications.delay(project_id=project.pk)
+            task_ids.append(task.id)
+        self.message_user(request, f"Started {len(task_ids)} tasks to delete classification: {task_ids}")
+
+    actions = [_remove_duplicate_classifications]
+
 
 @admin.register(Deployment)
 class DeploymentAdmin(admin.ModelAdmin[Deployment]):

ami/main/api/views.py

@@ -931,6 +931,9 @@ def zero_occurrences(self, queryset: QuerySet) -> QuerySet:
     def get_queryset(self) -> QuerySet:
         qs = super().get_queryset()
 
+        # First filter out taxa that have no occurrences
+        # qs = qs.filter(occurrences__isnull=False).distinct()
+
         occurrences_filter, occurrences_count_filter = self.get_occurrences_filters(qs)
 
         qs = qs.select_related("parent")
@@ -944,6 +947,8 @@ def get_queryset(self) -> QuerySet:
             if filter_active:
                 qs = self.filter_by_classification_threshold(qs)
                 qs = self.add_occurrence_counts(qs, occurrences_count_filter)
+                # Filter out taxa that have no occurrences or occurrences count is null
+                qs = qs.filter(occurrences_count__gt=0).filter(occurrences_count__isnull=False)
             else:
                 # If no filter don't return anything related to occurrences
                 # in a list view.

ami/main/models.py

@@ -560,13 +560,14 @@ def update_children(self):
         ]
         for model_name in child_models:
             model = apps.get_model("main", model_name)
-            project_values = set(model.objects.filter(deployment=self).values_list("project", flat=True).distinct())
-            if len(project_values) > 1:
+            qs = model.objects.filter(deployment=self).exclude(project=self.project)
+            project_values = set(qs.values_list("project", flat=True).distinct())
+            if len(project_values):
                 logger.warning(
                     f"Deployment {self} has alternate projects set on {model_name} "
                     f"objects: {project_values}. Updating them!"
                 )
-            model.objects.filter(deployment=self).exclude(project=self.project).update(project=self.project)
+            qs.update(project=self.project)
 
     def update_calculated_fields(self, save=False):
         """Update calculated fields on the deployment."""
@@ -601,15 +602,19 @@ def update_calculated_fields(self, save=False):
             self.save(update_calculated_fields=False)
 
     def save(self, update_calculated_fields=True, *args, **kwargs):
+        last_updated = self.updated_at or timezone.now()
         super().save(*args, **kwargs)
         if self.pk and update_calculated_fields:
+            # @TODO Use "dirty" flag strategy to only update when needed
+            new_or_updated_captures = self.captures.filter(updated_at__gte=last_updated).count()
+            deleted_captures = True if self.captures.count() < (self.captures_count or 0) else False
+            if new_or_updated_captures or deleted_captures:
+                ami.tasks.regroup_events.delay(self.pk)
             self.update_calculated_fields(save=True)
             if self.project:
                 self.update_children()
                 # @TODO this isn't working as a background task
                 # ami.tasks.model_task.delay("Project", self.project.pk, "update_children_project")
-            # @TODO Use "dirty" flag strategy to only update when needed
-            ami.tasks.regroup_events.delay(self.pk)
 
 
 @final
@@ -848,8 +853,7 @@ def group_images_into_events(
             [f'{d.strftime("%Y-%m-%d %H:%M:%S")} x{c}' for d, c in dupes.values_list("timestamp", "count")]
         )
         logger.warning(
-            f"Found multiple images with the same timestamp in deployment '{deployment}':\n "
-            f"{values}\n"
+            f"Found {len(values)} images with the same timestamp in deployment '{deployment}'. "
             f"Only one image will be used for each timestamp for each event."
         )
 
@@ -1696,6 +1700,8 @@ class Classification(BaseModel):
     )
     # job = models.CharField(max_length=255, null=True)
 
+    objects = ClassificationManager()
+
     # Type hints for auto-generated fields
     taxon_id: int
     algorithm_id: int

ami/ml/management/commands/remove_duplicate_classifications.py

@@ -0,0 +1,29 @@
+from django.core.management.base import BaseCommand
+from tqdm import tqdm
+
+from ami.main.models import Classification
+
+
+class Command(BaseCommand):
+    help = "Find and remove duplicate classifications on detections"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--project", type=int, help="Project ID to process")
+        parser.add_argument("--dry-run", action="store_true", help="List duplicates without deleting them")
+
+    def handle(self, *args, **options):
+        project_id = options["project"]
+        dry_run = options["dry_run"]
+
+        duplicates = Classification.objects.find_duplicates(project_id=project_id)  # type: ignore
+        total = duplicates.count()
+        self.stdout.write(f"Found {total} duplicate classifications")
+        if dry_run:
+            with tqdm(total=total, desc="Listing duplicates", unit="classification") as pbar:
+                for duplicate in duplicates:
+                    self.stdout.write(f"Duplicate classification: {duplicate}")
+                    pbar.update(1)
+
+        else:
+            with tqdm(total=None, desc="Deleting duplicates") as pbar:
+                duplicates.delete()

ami/ml/tasks.py

@@ -48,3 +48,30 @@ def create_detection_images(source_image_ids: list[int]):
             logger.debug(f"Created {len(processed_paths)} detection images for SourceImage #{source_image.pk}")
         except Exception as e:
             logger.error(f"Error creating detection images for SourceImage {source_image.pk}: {str(e)}")
+
+
+@celery_app.task(soft_time_limit=default_soft_time_limit, time_limit=default_time_limit)
+def remove_duplicate_classifications(project_id: int | None = None, dry_run: bool = False) -> int:
+    """
+    Remove duplicate classifications from the database.
+
+    A duplicate classification is one where the same detection, taxon, algorithm, score, softmax_output and raw_output
+    have been classified more than once. This can happen if the same detection is classified multiple times by the same
+    algorithm with the same result.
+
+    This method will keep the oldest classification and delete the rest.
+    """
+    from ami.main.models import Classification
+
+    # Find the oldest classification for each unique combination
+    duplicates_to_delete = Classification.objects.find_duplicates(project_id=project_id)  # type: ignore
+    num = duplicates_to_delete.count()
+
+    logger.info(f"Found {duplicates_to_delete.count()} duplicate classifications to delete")
+    if dry_run:
+        logger.info(f"Would delete {num} duplicate classifications")
+    else:
+        num_deleted = duplicates_to_delete.delete()
+        logger.info(f"Deleted {num_deleted} duplicate classifications")
+
+    return num_deleted