Merge branch 'main' into feat/update-identifications

Author: annavik

ami/jobs/apps.py

@@ -2,6 +2,6 @@
 from django.utils.translation import gettext_lazy as _
 
 
-class UsersConfig(AppConfig):
+class JobsConfig(AppConfig):
     name = "ami.jobs"
     verbose_name = _("Jobs")

ami/jobs/models.py

@@ -399,10 +399,9 @@ def run(cls, job: "Job"):
         total_detections = 0
         total_classifications = 0
 
-        # Set to low size because our response JSON just got enormous
-        # @TODO make this configurable
-        CHUNK_SIZE = 1
-        chunks = [images[i : i + CHUNK_SIZE] for i in range(0, image_count, CHUNK_SIZE)]  # noqa
+        config = job.pipeline.get_config(project_id=job.project.pk)
+        chunk_size = config.get("request_source_image_batch_size", 1)
+        chunks = [images[i : i + chunk_size] for i in range(0, image_count, chunk_size)]  # noqa
         request_failed_images = []
 
         for i, chunk in enumerate(chunks):
@@ -434,9 +433,9 @@ def run(cls, job: "Job"):
                 "process",
                 status=JobState.STARTED,
                 progress=(i + 1) / len(chunks),
-                processed=min((i + 1) * CHUNK_SIZE, image_count),
+                processed=min((i + 1) * chunk_size, image_count),
                 failed=len(request_failed_images),
-                remaining=max(image_count - ((i + 1) * CHUNK_SIZE), 0),
+                remaining=max(image_count - ((i + 1) * chunk_size), 0),
             )
 
             # count the completed, successful, and failed save_tasks:

ami/labelstudio/apps.py

@@ -2,6 +2,6 @@
 from django.utils.translation import gettext_lazy as _
 
 
-class UsersConfig(AppConfig):
+class LabelStudioConfig(AppConfig):
     name = "ami.labelstudio"
     verbose_name = _("Label Studio Integration")

ami/main/admin.py

@@ -517,6 +517,13 @@ class TaxaListAdmin(admin.ModelAdmin[TaxaList]):
     def taxa_count(self, obj) -> int:
         return obj.taxa.count()
 
+    autocomplete_fields = (
+        "taxa",
+        "projects",
+    )
+
+    list_filter = ("projects",)
+
 
 @admin.register(Device)
 class DeviceAdmin(admin.ModelAdmin[Device]):

ami/main/api/serializers.py

@@ -30,6 +30,7 @@
     SourceImage,
     SourceImageCollection,
     SourceImageUpload,
+    TaxaList,
     Taxon,
 )
 
@@ -532,6 +533,25 @@ def get_occurrences(self, obj):
         )
 
 
+class TaxaListSerializer(serializers.ModelSerializer):
+    taxa = serializers.SerializerMethodField()
+    projects = serializers.PrimaryKeyRelatedField(queryset=Project.objects.all(), many=True)
+
+    class Meta:
+        model = TaxaList
+        fields = ["id", "name", "description", "taxa", "projects"]
+
+    def get_taxa(self, obj):
+        """
+        Return URL to the taxa endpoint filtered by this taxalist.
+        """
+        return reverse_with_params(
+            "taxon-list",
+            request=self.context.get("request"),
+            params={"taxa_list_id": obj.pk},
+        )
+
+
 class CaptureTaxonSerializer(DefaultSerializer):
     parent = TaxonNoParentNestedSerializer(read_only=True)
     parents = TaxonParentSerializer(many=True, read_only=True)

ami/main/api/views.py

@@ -5,7 +5,7 @@
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core import exceptions
 from django.db import models
-from django.db.models import Prefetch
+from django.db.models import Prefetch, Q
 from django.db.models.functions import Coalesce
 from django.db.models.query import QuerySet
 from django.forms import BooleanField, CharField, IntegerField
@@ -59,6 +59,7 @@
     SourceImage,
     SourceImageCollection,
     SourceImageUpload,
+    TaxaList,
     Taxon,
     User,
     update_detection_counts,
@@ -89,6 +90,7 @@
     SourceImageUploadSerializer,
     StorageSourceSerializer,
     StorageStatusSerializer,
+    TaxaListSerializer,
     TaxonListSerializer,
     TaxonSearchResultSerializer,
     TaxonSerializer,
@@ -967,6 +969,38 @@ def filter_queryset(self, request, queryset, view):
         return queryset
 
 
+class OccurrenceTaxaListFilter(filters.BaseFilterBackend):
+    """
+    Filters occurrences based on a TaxaList.
+
+    Queries for all occurrences where the determination taxon is either:
+    - Directly in the requested TaxaList.
+    - A descendant (child or deeper) of any taxon in the TaxaList, recursively.
+
+    """
+
+    query_param = "taxa_list_id"
+
+    def filter_queryset(self, request, queryset, view):
+        taxalist_id = IntegerField(required=False).clean(request.query_params.get(self.query_param))
+        if taxalist_id:
+            taxa_list = TaxaList.objects.filter(id=taxalist_id).first()
+            if taxa_list:
+                taxa = taxa_list.taxa.all()  # Get taxalist taxon objects
+
+                # filter by the exact determination
+                query_filter = Q(determination__in=taxa)
+
+                # filter by the taxon's children
+                for taxon in taxa:
+                    query_filter |= Q(determination__parents_json__contains=[{"id": taxon.pk}])
+
+                queryset = queryset.filter(query_filter)
+                return queryset
+
+        return queryset
+
+
 class TaxonCollectionFilter(filters.BaseFilterBackend):
     """
     Filter taxa by the collection their occurrences belong to.
@@ -999,6 +1033,7 @@ class OccurrenceViewSet(DefaultViewSet, ProjectMixin):
         OccurrenceDateFilter,
         OccurrenceVerified,
         OccurrenceVerifiedByMeFilter,
+        OccurrenceTaxaListFilter,
     ]
     filterset_fields = [
         "event",
@@ -1030,9 +1065,9 @@ def get_serializer_class(self):
         else:
             return OccurrenceSerializer
 
-    def get_queryset(self) -> QuerySet:
+    def get_queryset(self) -> QuerySet["Occurrence"]:
         project = self.get_active_project()
-        qs = super().get_queryset()
+        qs = super().get_queryset().valid()  # type: ignore
         if project:
             qs = qs.filter(project=project)
         qs = qs.select_related(
@@ -1046,10 +1081,7 @@ def get_queryset(self) -> QuerySet:
         if self.action == "list":
             qs = (
                 qs.all()
-                .exclude(detections=None)
-                .exclude(event=None)
                 .filter(determination_score__gte=get_active_classification_threshold(self.request))
-                .exclude(first_appearance_timestamp=None)  # This must come after annotations
                 .order_by("-determination_score")
             )
 
@@ -1067,14 +1099,45 @@ def list(self, request, *args, **kwargs):
         return super().list(request, *args, **kwargs)
 
 
+class TaxonTaxaListFilter(filters.BaseFilterBackend):
+    """
+    Filters taxa based on a TaxaList Similar to `OccurrenceTaxaListFilter`.
+
+    Queries for all taxa that are either:
+    - Directly in the requested TaxaList.
+    - A descendant (child or deeper) of any taxon in the TaxaList, recursively.
+    """
+
+    query_param = "taxa_list_id"
+
+    def filter_queryset(self, request, queryset, view):
+        taxalist_id = IntegerField(required=False).clean(request.query_params.get(self.query_param))
+        if taxalist_id:
+            taxa_list = TaxaList.objects.filter(id=taxalist_id).first()
+            if taxa_list:
+                taxa = taxa_list.taxa.all()  # Get taxa in the TaxaList
+                query_filter = Q(id__in=taxa)
+                for taxon in taxa:
+                    query_filter |= Q(parents_json__contains=[{"id": taxon.pk}])
+
+                queryset = queryset.filter(query_filter)
+                return queryset
+
+        return queryset
+
+
 class TaxonViewSet(DefaultViewSet, ProjectMixin):
     """
     API endpoint that allows taxa to be viewed or edited.
     """
 
     queryset = Taxon.objects.all().defer("notes")
     serializer_class = TaxonSerializer
-    filter_backends = DefaultViewSetMixin.filter_backends + [CustomTaxonFilter, TaxonCollectionFilter]
+    filter_backends = DefaultViewSetMixin.filter_backends + [
+        CustomTaxonFilter,
+        TaxonCollectionFilter,
+        TaxonTaxaListFilter,
+    ]
     filterset_fields = [
         "name",
         "rank",
@@ -1286,6 +1349,19 @@ def list(self, request, *args, **kwargs):
         return super().list(request, *args, **kwargs)
 
 
+class TaxaListViewSet(viewsets.ModelViewSet, ProjectMixin):
+    queryset = TaxaList.objects.all()
+
+    def get_queryset(self):
+        qs = super().get_queryset()
+        project = self.get_active_project()
+        if project:
+            return qs.filter(projects=project)
+        return qs
+
+    serializer_class = TaxaListSerializer
+
+
 class ClassificationViewSet(DefaultViewSet, ProjectMixin):
     """
     API endpoint for viewing and adding classification results from a model.
@@ -1343,11 +1419,7 @@ def get(self, request):
                 "events_count": Event.objects.filter(deployment__project=project, deployment__isnull=False).count(),
                 "captures_count": SourceImage.objects.filter(deployment__project=project).count(),
                 # "detections_count": Detection.objects.filter(occurrence__project=project).count(),
-                "occurrences_count": Occurrence.objects.filter(
-                    project=project,
-                    # determination_score__gte=confidence_threshold,
-                    event__isnull=False,
-                ).count(),
+                "occurrences_count": Occurrence.objects.valid().filter(project=project).count(),  # type: ignore
                 "taxa_count": Occurrence.objects.all().unique_taxa(project=project).count(),  # type: ignore
             }
         else:
@@ -1357,10 +1429,7 @@ def get(self, request):
                 "events_count": Event.objects.filter(deployment__isnull=False).count(),
                 "captures_count": SourceImage.objects.count(),
                 # "detections_count": Detection.objects.count(),
-                "occurrences_count": Occurrence.objects.filter(
-                    # determination_score__gte=confidence_threshold,
-                    event__isnull=False
-                ).count(),
+                "occurrences_count": Occurrence.objects.valid().count(),  # type: ignore
                 "taxa_count": Occurrence.objects.all().unique_taxa().count(),  # type: ignore
                 "last_updated": timezone.now(),
             }

ami/main/models.py

@@ -2174,11 +2174,14 @@ def __str__(self) -> str:
         return f"#{self.pk} from SourceImage #{self.source_image_id} with Algorithm #{self.detection_algorithm_id}"
 
 
-class OccurrenceQuerySet(models.QuerySet):
-    def with_detections_count(self) -> models.QuerySet:
+class OccurrenceQuerySet(models.QuerySet["Occurrence"]):
+    def valid(self):
+        return self.exclude(detections__isnull=True)
+
+    def with_detections_count(self):
         return self.annotate(detections_count=models.Count("detections", distinct=True))
 
-    def with_timestamps(self) -> models.QuerySet:
+    def with_timestamps(self):
         """
         These are timestamps used for filtering and ordering in the UI.
         """
@@ -2192,14 +2195,14 @@ def with_timestamps(self) -> models.QuerySet:
             ),
         )
 
-    def with_identifications(self) -> models.QuerySet:
+    def with_identifications(self):
         return self.prefetch_related(
             "identifications",
             "identifications__taxon",
             "identifications__user",
         )
 
-    def unique_taxa(self, project: Project | None = None) -> models.QuerySet:
+    def unique_taxa(self, project: Project | None = None):
         qs = self
         if project:
             qs = self.filter(project=project)
@@ -2211,12 +2214,16 @@ def unique_taxa(self, project: Project | None = None) -> models.QuerySet:
         return qs
 
 
-class OccurrenceManager(models.Manager):
-    def get_queryset(self) -> OccurrenceQuerySet:
-        return OccurrenceQuerySet(self.model, using=self._db).select_related(
-            "determination",
-            "deployment",
-            "project",
+class OccurrenceManager(models.Manager.from_queryset(OccurrenceQuerySet)):
+    def get_queryset(self):
+        return (
+            super()
+            .get_queryset()
+            .select_related(
+                "determination",
+                "deployment",
+                "project",
+            )
         )
 
 
@@ -2236,7 +2243,7 @@ class Occurrence(BaseModel):
     detections: models.QuerySet[Detection]
     identifications: models.QuerySet[Identification]
 
-    objects: OccurrenceManager = OccurrenceManager()
+    objects = OccurrenceManager()
 
     def __str__(self) -> str:
         name = f"Occurrence #{self.pk}"

ami/ml/apps.py

@@ -2,6 +2,6 @@
 from django.utils.translation import gettext_lazy as _
 
 
-class UsersConfig(AppConfig):
+class MLConfig(AppConfig):
     name = "ami.ml"
     verbose_name = _("Machine Learning")

ami/ml/migrations/0021_pipeline_default_config.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.10 on 2025-03-19 16:27
+
+import ami.ml.schemas
+from django.db import migrations
+import django_pydantic_field.fields
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ml", "0020_projectpipelineconfig_alter_pipeline_projects"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pipeline",
+            name="default_config",
+            field=django_pydantic_field.fields.PydanticSchemaField(
+                config=None,
+                default=dict,
+                help_text="The default configuration for the pipeline. Used by both the job sending images to the pipeline and the processing service.",
+                schema=ami.ml.schemas.PipelineRequestConfigParameters,
+            ),
+        ),
+    ]