Merge branch 'main' into feat/adc-importer

Author: mihow

.envs/.ci/.django

@@ -16,3 +16,6 @@ MINIO_DEFAULT_BUCKET=ami-ci
 MINIO_STORAGE_USE_HTTPS=False
 MINIO_TEST_BUCKET=ami-test-ci
 MINIO_BROWSER_REDIRECT_URL=http://minio:9001
+
+DEFAULT_PROCESSING_SERVICE_NAME=Test Processing Service
+DEFAULT_PROCESSING_SERVICE_ENDPOINT=http://ml_backend:2000

ami/base/views.py

@@ -22,20 +22,23 @@ class ProjectMixin:
     def get_active_project(self) -> Project:
         from ami.base.serializers import SingleParamSerializer
 
+        param = "project_id"
+
         project_id = None
         # Extract from URL `/projects/` is in the url path
         if "/projects/" in self.request.path:
             project_id = self.kwargs.get("pk")
 
         # If not in URL, try query parameters
         if not project_id:
-            if self.require_project:
-                project_id = SingleParamSerializer[int].clean(
-                    param_name="project_id",
-                    field=serializers.IntegerField(required=True, min_value=0),
-                    data=self.request.query_params,
-                )
-            else:
-                project_id = self.request.query_params.get("project_id")  # No validation
+            # Look for project_id in GET query parameters or POST data
+            # POST data returns a list of ints, but QueryDict.get() returns a single value
+            project_id = self.request.query_params.get(param) or self.request.data.get(param)
+
+            project_id = SingleParamSerializer[int].clean(
+                param_name=param,
+                field=serializers.IntegerField(required=self.require_project, min_value=0),
+                data={param: project_id} if project_id else {},
+            )
 
         return get_object_or_404(Project, id=project_id) if project_id else None

ami/main/admin.py

@@ -76,6 +76,7 @@ def save_related(self, request, form, formsets, change):
     filter_horizontal = ("members",)
 
     inlines = [ProjectPipelineConfigInline]
+    autocomplete_fields = ("default_filters_include_taxa", "default_filters_exclude_taxa")
 
     fieldsets = (
         (
@@ -90,6 +91,18 @@ def save_related(self, request, form, formsets, change):
                 )
             },
         ),
+        (
+            "Settings",
+            {
+                "fields": (
+                    "default_processing_pipeline",
+                    "session_time_gap_seconds",
+                    "default_filters_score_threshold",
+                    "default_filters_include_taxa",
+                    "default_filters_exclude_taxa",
+                ),
+            },
+        ),
         (
             "Ownership & Access",
             {

ami/main/api/serializers.py

@@ -1,17 +1,16 @@
 import datetime
 
-from django.core.exceptions import ValidationError as DjangoValidationError
 from django.db.models import QuerySet
 from guardian.shortcuts import get_perms
 from rest_framework import serializers
 from rest_framework.request import Request
 
 from ami.base.fields import DateStringField
-from ami.base.serializers import DefaultSerializer, MinimalNestedModelSerializer, get_current_user, reverse_with_params
+from ami.base.serializers import DefaultSerializer, MinimalNestedModelSerializer, reverse_with_params
 from ami.jobs.models import Job
-from ami.main.models import Tag, create_source_image_from_upload
-from ami.ml.models import Algorithm
-from ami.ml.serializers import AlgorithmSerializer
+from ami.main.models import Tag
+from ami.ml.models import Algorithm, Pipeline
+from ami.ml.serializers import AlgorithmSerializer, PipelineNestedSerializer
 from ami.users.models import User
 from ami.users.roles import ProjectManager
 
@@ -25,14 +24,14 @@
     Occurrence,
     Page,
     Project,
+    ProjectSettingsMixin,
     S3StorageSource,
     Site,
     SourceImage,
     SourceImageCollection,
     SourceImageUpload,
     TaxaList,
     Taxon,
-    validate_filename_timestamp,
 )
 
 
@@ -255,6 +254,18 @@ class Meta:
         ]
 
 
+class TaxonNoParentNestedSerializer(DefaultSerializer):
+    class Meta:
+        model = Taxon
+        fields = [
+            "id",
+            "name",
+            "rank",
+            "details",
+            "gbif_taxon_key",
+        ]
+
+
 class ProjectListSerializer(DefaultSerializer):
     deployments_count = serializers.IntegerField(read_only=True)
 
@@ -272,10 +283,46 @@ class Meta:
         ]
 
 
+class ProjectSettingsSerializer(DefaultSerializer):
+    default_processing_pipeline = PipelineNestedSerializer(read_only=True)
+    default_processing_pipeline_id = serializers.PrimaryKeyRelatedField(
+        queryset=Pipeline.objects.all(),
+        source="default_processing_pipeline",
+        write_only=True,
+        required=False,
+        allow_null=True,
+    )
+    default_filters_include_taxa = TaxonNoParentNestedSerializer(read_only=True, many=True)
+    default_filters_include_taxa_ids = serializers.PrimaryKeyRelatedField(
+        queryset=Taxon.objects.all(),
+        many=True,
+        source="default_filters_include_taxa",
+        write_only=True,
+        required=False,
+    )
+    default_filters_exclude_taxa = TaxonNoParentNestedSerializer(read_only=True, many=True)
+    default_filters_exclude_taxa_ids = serializers.PrimaryKeyRelatedField(
+        queryset=Taxon.objects.all(),
+        many=True,
+        source="default_filters_exclude_taxa",
+        write_only=True,
+        required=False,
+    )
+
+    class Meta:
+        model = Project
+        fields = ProjectSettingsMixin.get_settings_field_names() + [
+            "default_processing_pipeline_id",
+            "default_filters_include_taxa_ids",
+            "default_filters_exclude_taxa_ids",
+        ]
+
+
 class ProjectSerializer(DefaultSerializer):
     deployments = DeploymentNestedSerializerWithLocationAndCounts(many=True, read_only=True)
     feature_flags = serializers.SerializerMethodField()
     owner = UserNestedSerializer(read_only=True)
+    settings = ProjectSettingsSerializer(source="*", required=False)
 
     def get_feature_flags(self, obj):
         if obj.feature_flags:
@@ -289,6 +336,7 @@ class Meta:
             "summary_data",  # @TODO move to a 2nd request, it's too slow
             "owner",
             "feature_flags",
+            "settings",
         ]
 
 
@@ -457,18 +505,6 @@ def get_occurrences(self, obj):
         )
 
 
-class TaxonNoParentNestedSerializer(DefaultSerializer):
-    class Meta:
-        model = Taxon
-        fields = [
-            "id",
-            "name",
-            "rank",
-            "details",
-            "gbif_taxon_key",
-        ]
-
-
 class TaxonParentSerializer(serializers.Serializer):
     id = serializers.IntegerField()
     name = serializers.CharField()
@@ -1047,30 +1083,6 @@ class Meta:
             "created_at",
         ]
 
-    def create(self, validated_data):
-        # Add the user to the validated data
-        request = self.context.get("request")
-        user = get_current_user(request)
-        # @TODO IMPORTANT ensure current user is a member of the deployment's project
-        obj = SourceImageUpload.objects.create(user=user, **validated_data)
-        source_image = create_source_image_from_upload(
-            obj.image,
-            obj.deployment,
-            request,
-        )
-        if source_image is not None:
-            obj.source_image = source_image  # type: ignore
-            obj.save()
-        return obj
-
-    def validate_image(self, value):
-        # Ensure that image filename contains a timestamp
-        try:
-            validate_filename_timestamp(value.name)
-        except DjangoValidationError as e:
-            raise serializers.ValidationError(str(e))
-        return value
-
 
 class SourceImageCollectionCommonKwargsSerializer(serializers.Serializer):
     # The most common kwargs for the sampling methods

ami/main/api/views.py

@@ -760,6 +760,7 @@ class SourceImageUploadViewSet(DefaultViewSet, ProjectMixin):
 
     serializer_class = SourceImageUploadSerializer
     permission_classes = [SourceImageUploadCRUDPermission]
+    require_project = True
 
     def get_queryset(self) -> QuerySet:
         # Only allow users to see their own uploads
@@ -772,6 +773,35 @@ def get_queryset(self) -> QuerySet:
     # This is the maximum limit for manually uploaded captures
     pagination_class.default_limit = 20
 
+    def perform_create(self, serializer):
+        """
+        Save the SourceImageUpload with the current user and create the associated SourceImage.
+        """
+        from ami.base.serializers import get_current_user
+        from ami.main.models import create_source_image_from_upload
+
+        # Get current user from request
+        user = get_current_user(self.request)
+        project = self.get_active_project()
+
+        # Create the SourceImageUpload object with the user
+        obj = serializer.save(user=user)
+
+        # Get process_now flag from project feature flags
+        process_now = project.feature_flags.auto_processs_manual_uploads
+
+        # Create source image from the upload
+        source_image = create_source_image_from_upload(
+            image=obj.image,
+            deployment=obj.deployment,
+            request=self.request,
+            process_now=process_now,
+        )
+
+        # Update the source_image reference and save
+        obj.source_image = source_image
+        obj.save()
+
 
 class DetectionViewSet(DefaultViewSet, ProjectMixin):
     """

ami/main/migrations/0064_alter_project_description.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.10 on 2025-08-08 15:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main", "0063_alter_sourceimagecollection_method"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="project",
+            name="description",
+            field=models.TextField(blank=True),
+        ),
+    ]

ami/main/migrations/0065_project_default_filters_exclude_taxa_and_more.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.10 on 2025-08-11 22:19
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ml", "0022_alter_pipeline_default_config"),
+        ("main", "0064_alter_project_description"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="project",
+            name="default_filters_exclude_taxa",
+            field=models.ManyToManyField(
+                blank=True,
+                help_text="Taxa that are excluded by default in the occurrence filters and metrics. For example, 'Not a Moth'.",
+                related_name="exclude_taxa_default_projects",
+                to="main.taxon",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="default_filters_include_taxa",
+            field=models.ManyToManyField(
+                blank=True,
+                help_text="Taxa that are included by default in the occurrence filters and metrics. For example, the top-level taxa like 'Moths' or 'Arthropods'. ",
+                related_name="include_taxa_default_projects",
+                to="main.taxon",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="default_filters_score_threshold",
+            field=models.FloatField(default=0.5, help_text="Default score threshold for filtering occurrences"),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="default_processing_pipeline",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="The default pipeline to use for processing images in this project. This is used to determine which processing service to run on new images. ",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="default_projects",
+                to="ml.pipeline",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="session_time_gap_seconds",
+            field=models.IntegerField(default=7200, help_text="Time gap in seconds to consider a new session"),
+        ),
+    ]

ami/main/migrations/0066_alter_project_feature_flags_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.10 on 2025-08-08 21:53
+
+import ami.main.models
+from django.db import migrations, models
+import django_pydantic_field.fields
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("main", "0065_project_default_filters_exclude_taxa_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="project",
+            name="feature_flags",
+            field=django_pydantic_field.fields.PydanticSchemaField(
+                blank=True,
+                config=None,
+                default={"auto_processs_manual_uploads": False, "tags": False},
+                schema=ami.main.models.ProjectFeatureFlags,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="sourceimageupload",
+            name="image",
+            field=models.ImageField(upload_to=ami.main.models.upload_to_with_deployment),
+        ),
+    ]