Make it possible to export data from UI

ami/exports/admin.py

@@ -13,13 +13,13 @@ class DataExportAdmin(admin.ModelAdmin):
     list_display = ("id", "user", "format", "status_display", "project", "created_at", "get_job")
     list_filter = ("format", "project")
     search_fields = ("user__username", "format", "project__name")
-    readonly_fields = ("status_display", "file_url_display")
+    readonly_fields = ("status_display", "file_url_display", "filters_display", "created_at", "updated_at")
 
     fieldsets = (
         (
             None,
             {
-                "fields": ("user", "format", "project", "filters"),
+                "fields": ("user", "format", "project", "filters", "filters_display"),
             },
         ),
         (
@@ -39,7 +39,7 @@ def get_queryset(self, request: HttpRequest):
 
     @admin.display(description="Status")
     def status_display(self, obj):
-        return obj.status  # Calls the @property from the model
+        return obj.job.status if obj.job else "No Job"
 
     @admin.display(description="File URL")
     def file_url_display(self, obj):

ami/exports/format_types.py

@@ -44,7 +44,8 @@ def get_serializer_class(self):
 
     def get_queryset(self):
         return (
-            Occurrence.objects.filter(project=self.project)
+            Occurrence.objects.valid()  # type: ignore[union-attr]  Custom manager method
+            .filter(project=self.project)
             .select_related(
                 "determination",
                 "deployment",
@@ -127,7 +128,8 @@ class CSVExporter(BaseExporter):
 
     def get_queryset(self):
         return (
-            Occurrence.objects.filter(project=self.project)
+            Occurrence.objects.valid()  # type: ignore[union-attr]  Custom queryset method
+            .filter(project=self.project)
             .select_related(
                 "determination",
                 "deployment",

ami/exports/migrations/0001_initial.py

@@ -24,7 +24,7 @@ class Migration(migrations.Migration):
                     "format",
                     models.CharField(
                         choices=[
-                            ("occurrences_simple_json", "occurrences_simple_json"),
+                            ("occurrences_api_json", "occurrences_api_json"),
                             ("occurrences_simple_csv", "occurrences_simple_csv"),
                         ],
                         max_length=255,

ami/exports/models.py

@@ -41,8 +41,8 @@ def get_filters_display(self):
         from django.apps import apps
 
         related_models = {
-            "collection": "main.SourceImageCollection",
-            "taxa_list": "main.TaxaList",
+            "collection_id": "main.SourceImageCollection",
+            "taxa_list_id": "main.TaxaList",
         }
         filters = self.filters or {}
         filters_display = {}
@@ -53,7 +53,9 @@ def get_filters_display(self):
                 try:
                     Model = apps.get_model(model_path)
                     instance = Model.objects.get(pk=value)
-                    filters_display[key] = {"id": value, "name": str(instance)}
+                    name = getattr(instance, "name", str(instance))
+                    key = key.replace("_id", "")  # Could potentially use the field name of the relationship instead
+                    filters_display[key] = {"id": value, "name": name}
                 except Model.DoesNotExist:
                     filters_display[key] = {"id": value, "name": f"{model_path} with id {value} not found"}
                 except Exception as e:
@@ -67,7 +69,9 @@ def generate_filename(self):
         """Generates a slugified filename using project name and export ID."""
         from ami.exports.registry import ExportRegistry
 
-        extension = ExportRegistry.get_exporter(self.format).file_format
+        registry = ExportRegistry.get_exporter(self.format)
+        assert registry, f"Export format '{self.format}' not found in registry"
+        extension = registry.file_format
         project_slug = slugify(self.project.name)  # Convert project name to a slug
         return f"{project_slug}_export-{self.pk}.{extension}"
 

ami/exports/registry.py

@@ -25,5 +25,5 @@ def get_supported_formats(cls):
         return list(cls._registry.keys())
 
 
-ExportRegistry.register("occurrences_simple_json")(format_types.JSONExporter)
+ExportRegistry.register("occurrences_api_json")(format_types.JSONExporter)
 ExportRegistry.register("occurrences_simple_csv")(format_types.CSVExporter)

ami/exports/tests.py

@@ -1,31 +1,43 @@
 import csv
 import json
+import logging
 
 from django.core.files.base import ContentFile
 from django.core.files.storage import default_storage
 from django.test import TestCase
 from rest_framework.test import APIClient
 
 from ami.exports.models import DataExport
-from ami.main.models import Deployment, Occurrence, Project, SourceImageCollection
-from ami.tests.fixtures.main import create_captures
-from ami.users.models import User
+from ami.main.models import Occurrence, SourceImageCollection
+from ami.tests.fixtures.main import (
+    create_captures,
+    create_occurrences,
+    create_taxa,
+    group_images_into_events,
+    setup_test_project,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class DataExportTest(TestCase):
     def setUp(self):
-        self.user = User.objects.create_user(email="testuser@insectai.org", is_superuser=True, is_staff=True)
-        self.project = Project.objects.create(name="Test Project")
+        self.project, self.deployment = setup_test_project(reuse=False)
+        self.user = self.project.owner
+        self.assertIsNotNone(self.user, "Project owner should not be None.")
         self.client = APIClient()
         self.client.force_authenticate(user=self.user)
-        # Create test deployment
-        self.deployment = Deployment.objects.create(name="Test Deployment", project=self.project)
-        # Create captures for the deployment
-        create_captures(deployment=self.deployment, num_nights=2, images_per_night=10, interval_minutes=1)
+        # Create captures & occurrences to test exporting
+        create_captures(deployment=self.deployment, num_nights=2, images_per_night=4, interval_minutes=1)
+        group_images_into_events(self.deployment)
+        create_taxa(self.project)
+        create_occurrences(num=10, deployment=self.deployment)
+        # Assert project has occurrences
+        self.assertGreater(self.project.occurrences.count(), 0, "No occurrences created for testing.")
         # Create a collection using the provided method
         self.collection = self._create_collection()
         # Define export formats
-        self.export_formats = ["occurrences_simple_csv", "occurrences_simple_json"]
+        self.export_formats = ["occurrences_simple_csv", "occurrences_api_json"]
 
     def _create_export_with_file(self, format_type):
         filename = f"exports/test_export_file_{format_type}.json"
@@ -54,14 +66,20 @@ def test_file_is_deleted_when_export_is_deleted(self):
 
     def _create_collection(self):
         """Create a SourceImageCollection from deployment captures."""
-        images = self.deployment.captures.all()
+        images = self.project.captures.all()
+        # Use only half of the images for the collection
+        collection_images = images[: images.count() // 2]
+
+        # Ensure collection images are fewer than total images
+        self.assertGreater(len(collection_images), 0, "No collection images to test exports.")
+        self.assertLess(len(collection_images), images.count(), "Collection images should be fewer than total images.")
 
         # Create the collection
         collection = SourceImageCollection.objects.create(
             name="Test Manual Source Image Collection",
             project=self.project,
             method="manual",
-            kwargs={"image_ids": [image.pk for image in images]},
+            kwargs={"image_ids": [image.pk for image in collection_images]},
         )
         collection.save()
 
@@ -76,7 +94,7 @@ def run_and_validate_export(self, format_type):
             user=self.user,
             project=self.project,
             format=format_type,
-            filters={"collection": self.collection.pk},
+            filters={"collection_id": self.collection.pk},
             job=None,
         )
 
@@ -92,29 +110,48 @@ def run_and_validate_export(self, format_type):
         with default_storage.open(file_path, "r") as f:
             if format_type == "occurrences_simple_csv":
                 self.validate_csv_records(f)
-            elif format_type == "occurrences_simple_json":
+            elif format_type == "occurrences_api_json":
                 self.validate_json_records(f)
 
         # Clean up the exported file after the test
         default_storage.delete(file_path)
 
+    def test_export_record_count(self):
+        """Test record count in the exported file."""
+        for format_type in self.export_formats:
+            with self.subTest(format=format_type):
+                self.run_and_validate_export(format_type)
+
+    def validate_record_count(self, record_count):
+        """Validate record count in the exported file."""
+        collection_count = (
+            Occurrence.objects.valid()  # type: ignore[union-attr] # Custom queryset method
+            .filter(detections__source_image__collections=self.collection)
+            .distinct()
+            .count()
+        )
+        total_count = Occurrence.objects.valid().filter(project=self.project).count()  # type: ignore[union-attr]
+
+        logger.debug(f"Exported: {record_count}, # in Collection: {collection_count}, # in Project: {total_count}")
+        self.assertGreater(record_count, 0, "Record count should be greater than zero.")
+        self.assertLess(record_count, total_count, "Record count should be less than total occurrences.")
+        self.assertEqual(record_count, collection_count, "Record count does not match expected count.")
+
     def validate_csv_records(self, file):
         """Validate record count in CSV."""
         csv_reader = csv.DictReader(file)
         row_count = sum(1 for row in csv_reader)
-        expected_count = Occurrence.objects.filter(detections__source_image__collections=self.collection).count()
-        self.assertEqual(row_count, expected_count)
+        self.validate_record_count(row_count)
 
     def validate_json_records(self, file):
         """Validate record count in JSON."""
         data = json.load(file)
-        expected_count = Occurrence.objects.filter(detections__source_image__collections=self.collection).count()
-        self.assertEqual(len(data), expected_count)
+        self.validate_record_count(len(data))
 
     def test_csv_export_record_count(self):
         """Test CSV export record count."""
         self.run_and_validate_export("occurrences_simple_csv")
 
     def test_json_export_record_count(self):
         """Test JSON export record count."""
-        self.run_and_validate_export("occurrences_simple_json")
+        self.run_and_validate_export("occurrences_api_json")

ami/main/api/views.py

@@ -809,9 +809,9 @@ class CustomTaxonFilter(filters.BaseFilterBackend):
 
     query_params = ["taxon"]
 
-    def get_filter_taxon(self, request: Request) -> Taxon | None:
+    def get_filter_taxon(self, request: Request, query_params: list[str] | None = None) -> Taxon | None:
         taxon_id = None
-        for param in self.query_params:
+        for param in query_params or self.query_params:
             taxon_id = request.query_params.get(param)
             if taxon_id:
                 break
@@ -846,7 +846,7 @@ class CustomOccurrenceDeterminationFilter(CustomTaxonFilter):
     query_params = ["determination", "taxon"]
 
     def filter_queryset(self, request, queryset, view):
-        taxon = self.get_filter_taxon(request)
+        taxon = self.get_filter_taxon(request, query_params=self.query_params)
         if taxon:
             # Here the queryset is the Occurrence queryset
             return queryset.filter(
@@ -861,10 +861,14 @@ class OccurrenceCollectionFilter(filters.BaseFilterBackend):
     Filter occurrences by the collection their detections source images belong to.
     """
 
-    query_param = "collection"
+    query_params = ["collection_id", "collection"]  # @TODO remove "collection" param when UI is updated
 
     def filter_queryset(self, request, queryset, view):
-        collection_id = IntegerField(required=False).clean(request.query_params.get(self.query_param))
+        collection_id = None
+        for param in self.query_params:
+            collection_id = IntegerField(required=False).clean(request.query_params.get(param))
+            if collection_id:
+                break
         if collection_id:
             # Here the queryset is the Occurrence queryset
             return queryset.filter(detections__source_image__collections=collection_id)

ami/main/models.py

@@ -125,7 +125,9 @@ class Project(BaseModel):
     image = models.ImageField(upload_to="projects", blank=True, null=True)
     owner = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, related_name="projects")
     members = models.ManyToManyField(User, related_name="user_projects", blank=True)
+
     # Backreferences for type hinting
+    captures: models.QuerySet["SourceImage"]
     deployments: models.QuerySet["Deployment"]
     events: models.QuerySet["Event"]
     occurrences: models.QuerySet["Occurrence"]
@@ -138,6 +140,7 @@ class Project(BaseModel):
     devices: models.QuerySet["Device"]
     sites: models.QuerySet["Site"]
     jobs: models.QuerySet["Job"]
+
     objects = ProjectManager()
 
     def get_project(self):

ami/tests/fixtures/main.py

@@ -11,7 +11,6 @@
 from ami.main.models import (
     Deployment,
     Detection,
-    Event,
     Occurrence,
     Project,
     SourceImage,
@@ -100,12 +99,15 @@ def create_test_project(name: str | None) -> Project:
     short_id = uuid.uuid4().hex[:8]
     name = name or f"Test Project {short_id}"
 
-    admin_user = User.objects.filter(is_superuser=True).first()
-    project = Project.objects.create(name=name, owner=admin_user, description="Test description")
-    data_source = create_storage_source(project, f"Test Data Source {short_id}", prefix=f"{short_id}")
-    create_deployment(project, data_source, f"Test Deployment {short_id}")
-    create_processing_service(project)
-    return project
+    with transaction.atomic():
+        admin_user, _ = User.objects.get_or_create(
+            email=f"antenna+{short_id}@insectai.org", is_superuser=True, is_staff=True
+        )
+        project = Project.objects.create(name=name, owner=admin_user, description="Test description")
+        data_source = create_storage_source(project, f"Test Data Source {short_id}", prefix=f"{short_id}")
+        create_deployment(project, data_source, f"Test Deployment {short_id}")
+        create_processing_service(project)
+        return project
 
 
 def setup_test_project(reuse=True) -> tuple[Project, Deployment]:
@@ -349,25 +351,28 @@ def create_occurrences(
     num: int = 6,
     taxon: Taxon | None = None,
 ):
-    event = Event.objects.filter(deployment=deployment).first()
-    if not event:
-        raise ValueError("No events found for deployment")
-
-    for i in range(num):
-        # Every Occurrence requires a Detection
-        source_image = SourceImage.objects.filter(event=event).order_by("?").first()
-        if not source_image:
-            raise ValueError("No source images found for event")
-        taxon = taxon or Taxon.objects.filter(projects=deployment.project).order_by("?").first()
+    # Get all source images for the deployment that have an event
+    source_images = list(SourceImage.objects.filter(deployment=deployment))
+    if not source_images:
+        raise ValueError("No source images with events found for deployment")
+
+    # Get taxon if not provided
+    if not taxon:
+        taxon = Taxon.objects.filter(projects=deployment.project).order_by("?").first()
         if not taxon:
             raise ValueError("No taxa found for project")
+
+    # Create occurrences evenly distributed across all source images
+    for i in range(num):
+        # Select images in a round-robin fashion
+        source_image = source_images[i % len(source_images)]
+
         detection = Detection.objects.create(
             source_image=source_image,
-            timestamp=source_image.timestamp,  # @TODO this should be automatically set to the source image timestamp
+            timestamp=source_image.timestamp,
             bbox=[0.1, 0.1, 0.2, 0.2],
         )
-        # Could speed this up by creating an Occurrence with a determined taxon directly
-        # but this tests more of the code.
+
         detection.classifications.create(
             taxon=taxon,
             score=0.9,

ui/src/app.tsx

@@ -24,6 +24,7 @@ import { Occurrences } from 'pages/occurrences/occurrences'
 import { Collections } from 'pages/project/collections/collections'
 import { Devices } from 'pages/project/entities/devices'
 import { Sites } from 'pages/project/entities/sites'
+import { Exports } from 'pages/project/exports/exports'
 import { General } from 'pages/project/general/general'
 import { Pipelines } from 'pages/project/pipelines/pipelines'
 import { ProcessingServices } from 'pages/project/processing-services/processing-services'
@@ -106,6 +107,7 @@ export const App = () => (
             <Route path="summary" element={<Summary />} />
             <Route path="collections" element={<Collections />} />
             <Route path="collections/:id" element={<CollectionDetails />} />
+            <Route path="exports/:id?" element={<Exports />} />
             <Route
               path="processing-services/:id?"
               element={<ProcessingServices />}