Remove default classification threshold (#613)

* feat: remove the default classification threshold on all queries

* chore: update formatting

Author: mihow

ami/main/api/views.py

@@ -374,12 +374,7 @@ class SourceImageViewSet(DefaultViewSet):
     GET /captures/1/
     """
 
-    queryset = (
-        SourceImage.objects.all()
-        .with_occurrences_count()  # type: ignore
-        .with_taxa_count()
-        # .with_detections_count()
-    )
+    queryset = SourceImage.objects.all()
 
     serializer_class = SourceImageSerializer
     filterset_fields = ["event", "deployment", "deployment__project", "collections"]
@@ -411,6 +406,13 @@ def get_queryset(self) -> QuerySet:
         queryset = super().get_queryset()
         with_detections_default = False
 
+        classification_threshold = get_active_classification_threshold(self.request)
+        queryset = queryset.with_occurrences_count(  # type: ignore
+            classification_threshold=classification_threshold
+        ).with_taxa_count(  # type: ignore
+            classification_threshold=classification_threshold
+        )
+
         queryset.select_related(
             "event",
             "deployment",
@@ -542,8 +544,6 @@ class SourceImageCollectionViewSet(DefaultViewSet):
         SourceImageCollection.objects.all()
         .with_source_images_count()  # type: ignore
         .with_source_images_with_detections_count()
-        .with_occurrences_count()
-        .with_taxa_count()
         .prefetch_related("jobs")
     )
     serializer_class = SourceImageCollectionSerializer
@@ -559,6 +559,16 @@ class SourceImageCollectionViewSet(DefaultViewSet):
         "occurrences_count",
     ]
 
+    def get_queryset(self) -> QuerySet:
+        classification_threshold = get_active_classification_threshold(self.request)
+        queryset = (
+            super()
+            .get_queryset()
+            .with_occurrences_count(classification_threshold=classification_threshold)  # type: ignore
+            .with_taxa_count(classification_threshold=classification_threshold)
+        )
+        return queryset
+
     @action(detail=True, methods=["post"], name="populate")
     def populate(self, request, pk=None):
         """

ami/main/models.py

@@ -580,7 +580,6 @@ def update_calculated_fields(self, save=False):
         self.detections_count = Detection.objects.filter(Q(source_image__deployment=self)).count()
         self.occurrences_count = (
             self.occurrences.filter(
-                determination_score__gte=settings.DEFAULT_CONFIDENCE_THRESHOLD,
                 event__isnull=False,
             )
             .distinct()
@@ -589,7 +588,6 @@ def update_calculated_fields(self, save=False):
         self.taxa_count = (
             Taxon.objects.filter(
                 occurrences__deployment=self,
-                occurrences__determination_score__gte=settings.DEFAULT_CONFIDENCE_THRESHOLD,
                 occurrences__event__isnull=False,
             )
             .distinct()
@@ -710,12 +708,8 @@ def get_captures_count(self) -> int:
     def get_detections_count(self) -> int | None:
         return Detection.objects.filter(Q(source_image__event=self)).count()
 
-    def get_occurrences_count(self, classification_threshold: int | None = None) -> int:
-        return (
-            self.occurrences.distinct()
-            .filter(determination_score__gte=classification_threshold or settings.DEFAULT_CONFIDENCE_THRESHOLD)
-            .count()
-        )
+    def get_occurrences_count(self, classification_threshold: float = 0) -> int:
+        return self.occurrences.distinct().filter(determination_score__gte=classification_threshold).count()
 
     def stats(self) -> dict[str, int | None]:
         return (
@@ -728,15 +722,15 @@ def stats(self) -> dict[str, int | None]:
             )
         )
 
-    def taxa_count(self, classification_threshold: int | None = None) -> int:
+    def taxa_count(self, classification_threshold: float = 0) -> int:
         # Move this to a pre-calculated field or prefetch_related in the view
         # return self.taxa(classification_threshold).count()
         return 0
 
-    def taxa(self, classification_threshold: int | None = None) -> models.QuerySet["Taxon"]:
+    def taxa(self, classification_threshold: float = 0) -> models.QuerySet["Taxon"]:
         return Taxon.objects.filter(
             Q(occurrences__event=self),
-            occurrences__determination_score__gte=classification_threshold or settings.DEFAULT_CONFIDENCE_THRESHOLD,
+            occurrences__determination_score__gte=classification_threshold,
         ).distinct()
 
     def first_capture(self):
@@ -1145,23 +1139,23 @@ def delete_source_image(sender, instance, **kwargs):
 
 
 class SourceImageQuerySet(models.QuerySet):
-    def with_occurrences_count(self):
+    def with_occurrences_count(self, classification_threshold: float = 0):
         return self.annotate(
             occurrences_count=models.Count(
                 "detections__occurrence",
                 filter=models.Q(
-                    detections__occurrence__determination_score__gte=settings.DEFAULT_CONFIDENCE_THRESHOLD
+                    detections__occurrence__determination_score__gte=classification_threshold,
                 ),
                 distinct=True,
             )
         )
 
-    def with_taxa_count(self):
+    def with_taxa_count(self, classification_threshold: float = 0):
         return self.annotate(
             taxa_count=models.Count(
                 "detections__occurrence__determination",
                 filter=models.Q(
-                    detections__occurrence__determination_score__gte=settings.DEFAULT_CONFIDENCE_THRESHOLD
+                    detections__occurrence__determination_score__gte=classification_threshold,
                 ),
                 distinct=True,
             )
@@ -2475,7 +2469,7 @@ def occurrence_images(
         self,
         limit: int | None = 10,
         project_id: int | None = None,
-        classification_threshold: float | None = None,
+        classification_threshold: float = 0,
     ) -> list[str]:
         """
         Return one image from each occurrence of this Taxon.
@@ -2489,8 +2483,6 @@ def occurrence_images(
         Use the request to generate the full media URLs.
         """
 
-        classification_threshold = classification_threshold or settings.DEFAULT_CONFIDENCE_THRESHOLD
-
         # Retrieve the URLs using a single optimized query
         qs = (
             self.occurrences.prefetch_related(
@@ -2663,22 +2655,25 @@ def with_source_images_with_detections_count(self):
             )
         )
 
-    def with_occurrences_count(self):
+    def with_occurrences_count(self, classification_threshold: float = 0):
         return self.annotate(
             occurrences_count=models.Count(
                 "images__detections__occurrence",
                 filter=models.Q(
-                    images__detections__occurrence__determination_score__gte=settings.DEFAULT_CONFIDENCE_THRESHOLD
+                    images__detections__occurrence__determination_score__gte=classification_threshold,
                 ),
                 distinct=True,
             )
         )
 
-    def with_taxa_count(self):
+    def with_taxa_count(self, classification_threshold: float = 0):
         return self.annotate(
             taxa_count=models.Count(
                 "images__detections__occurrence__determination",
                 distinct=True,
+                filter=models.Q(
+                    images__detections__occurrence__determination_score__gte=classification_threshold,
+                ),
             )
         )
 

ami/utils/requests.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.forms import FloatField
 from rest_framework.request import Request
 
@@ -10,5 +9,5 @@ def get_active_classification_threshold(request: Request) -> float:
     if classification_threshold is not None:
         classification_threshold = FloatField(required=False).clean(classification_threshold)
     else:
-        classification_threshold = settings.DEFAULT_CONFIDENCE_THRESHOLD
+        classification_threshold = 0
     return classification_threshold

config/settings/base.py

@@ -369,8 +369,6 @@
 # Your stuff...
 # ------------------------------------------------------------------------------
 
-DEFAULT_CONFIDENCE_THRESHOLD = env.float("DEFAULT_CONFIDENCE_THRESHOLD", default=0.6)  # type: ignore[no-untyped-call]
-
 S3_TEST_ENDPOINT = env("MINIO_ENDPOINT", default="http://minio:9000")  # type: ignore[no-untyped-call]
 S3_TEST_KEY = env("MINIO_ROOT_USER", default=None)  # type: ignore[no-untyped-call]
 S3_TEST_SECRET = env("MINIO_ROOT_PASSWORD", default=None)  # type: ignore[no-untyped-call]