Storage and retrieval of project settings (#918)

* chore: separate fields & type hint backreferences in Project model

* feat: basic method to store user-defined settings for projects

* fix: don't fail whole migration if create roles task fails

* fix: admin form with too many choices

* feat: return nested representations of related models in settings

Author: mihow

ami/main/admin.py

@@ -76,6 +76,7 @@ def save_related(self, request, form, formsets, change):
     filter_horizontal = ("members",)
 
     inlines = [ProjectPipelineConfigInline]
+    autocomplete_fields = ("default_filters_include_taxa", "default_filters_exclude_taxa")
 
     fieldsets = (
         (
@@ -90,6 +91,18 @@ def save_related(self, request, form, formsets, change):
                 )
             },
         ),
+        (
+            "Settings",
+            {
+                "fields": (
+                    "default_processing_pipeline",
+                    "session_time_gap_seconds",
+                    "default_filters_score_threshold",
+                    "default_filters_include_taxa",
+                    "default_filters_exclude_taxa",
+                ),
+            },
+        ),
         (
             "Ownership & Access",
             {

ami/main/api/serializers.py

@@ -10,8 +10,8 @@
 from ami.base.serializers import DefaultSerializer, MinimalNestedModelSerializer, get_current_user, reverse_with_params
 from ami.jobs.models import Job
 from ami.main.models import Tag, create_source_image_from_upload
-from ami.ml.models import Algorithm
-from ami.ml.serializers import AlgorithmSerializer
+from ami.ml.models import Algorithm, Pipeline
+from ami.ml.serializers import AlgorithmSerializer, PipelineNestedSerializer
 from ami.users.models import User
 from ami.users.roles import ProjectManager
 
@@ -25,6 +25,7 @@
     Occurrence,
     Page,
     Project,
+    ProjectSettingsMixin,
     S3StorageSource,
     Site,
     SourceImage,
@@ -255,6 +256,18 @@ class Meta:
         ]
 
 
+class TaxonNoParentNestedSerializer(DefaultSerializer):
+    class Meta:
+        model = Taxon
+        fields = [
+            "id",
+            "name",
+            "rank",
+            "details",
+            "gbif_taxon_key",
+        ]
+
+
 class ProjectListSerializer(DefaultSerializer):
     deployments_count = serializers.IntegerField(read_only=True)
 
@@ -272,10 +285,46 @@ class Meta:
         ]
 
 
+class ProjectSettingsSerializer(DefaultSerializer):
+    default_processing_pipeline = PipelineNestedSerializer(read_only=True)
+    default_processing_pipeline_id = serializers.PrimaryKeyRelatedField(
+        queryset=Pipeline.objects.all(),
+        source="default_processing_pipeline",
+        write_only=True,
+        required=False,
+        allow_null=True,
+    )
+    default_filters_include_taxa = TaxonNoParentNestedSerializer(read_only=True, many=True)
+    default_filters_include_taxa_ids = serializers.PrimaryKeyRelatedField(
+        queryset=Taxon.objects.all(),
+        many=True,
+        source="default_filters_include_taxa",
+        write_only=True,
+        required=False,
+    )
+    default_filters_exclude_taxa = TaxonNoParentNestedSerializer(read_only=True, many=True)
+    default_filters_exclude_taxa_ids = serializers.PrimaryKeyRelatedField(
+        queryset=Taxon.objects.all(),
+        many=True,
+        source="default_filters_exclude_taxa",
+        write_only=True,
+        required=False,
+    )
+
+    class Meta:
+        model = Project
+        fields = ProjectSettingsMixin.get_settings_field_names() + [
+            "default_processing_pipeline_id",
+            "default_filters_include_taxa_ids",
+            "default_filters_exclude_taxa_ids",
+        ]
+
+
 class ProjectSerializer(DefaultSerializer):
     deployments = DeploymentNestedSerializerWithLocationAndCounts(many=True, read_only=True)
     feature_flags = serializers.SerializerMethodField()
     owner = UserNestedSerializer(read_only=True)
+    settings = ProjectSettingsSerializer(source="*", required=False)
 
     def get_feature_flags(self, obj):
         if obj.feature_flags:
@@ -289,6 +338,7 @@ class Meta:
             "summary_data",  # @TODO move to a 2nd request, it's too slow
             "owner",
             "feature_flags",
+            "settings",
         ]
 
 
@@ -457,18 +507,6 @@ def get_occurrences(self, obj):
         )
 
 
-class TaxonNoParentNestedSerializer(DefaultSerializer):
-    class Meta:
-        model = Taxon
-        fields = [
-            "id",
-            "name",
-            "rank",
-            "details",
-            "gbif_taxon_key",
-        ]
-
-
 class TaxonParentSerializer(serializers.Serializer):
     id = serializers.IntegerField()
     name = serializers.CharField()

ami/main/migrations/0065_project_default_filters_exclude_taxa_and_more.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.10 on 2025-08-11 22:19
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ml", "0022_alter_pipeline_default_config"),
+        ("main", "0064_alter_project_description"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="project",
+            name="default_filters_exclude_taxa",
+            field=models.ManyToManyField(
+                blank=True,
+                help_text="Taxa that are excluded by default in the occurrence filters and metrics. For example, 'Not a Moth'.",
+                related_name="exclude_taxa_default_projects",
+                to="main.taxon",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="default_filters_include_taxa",
+            field=models.ManyToManyField(
+                blank=True,
+                help_text="Taxa that are included by default in the occurrence filters and metrics. For example, the top-level taxa like 'Moths' or 'Arthropods'. ",
+                related_name="include_taxa_default_projects",
+                to="main.taxon",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="default_filters_score_threshold",
+            field=models.FloatField(default=0.5, help_text="Default score threshold for filtering occurrences"),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="default_processing_pipeline",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="The default pipeline to use for processing images in this project. This is used to determine which processing service to run on new images. ",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="default_projects",
+                to="ml.pipeline",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="session_time_gap_seconds",
+            field=models.IntegerField(default=7200, help_text="Time gap in seconds to consider a new session"),
+        ),
+    ]

ami/main/models.py

@@ -29,6 +29,7 @@
 from ami.base.fields import DateStringField
 from ami.base.models import BaseModel
 from ami.main import charts
+from ami.main.models_future.projects import ProjectSettingsMixin
 from ami.users.models import User
 from ami.utils.schemas import OrderedEnum
 
@@ -201,32 +202,32 @@ class ProjectFeatureFlags(pydantic.BaseModel):
 
 
 @final
-class Project(BaseModel):
+class Project(ProjectSettingsMixin, BaseModel):
     """ """
 
     name = models.CharField(max_length=_POST_TITLE_MAX_LENGTH)
     description = models.TextField(blank=True)
     image = models.ImageField(upload_to="projects", blank=True, null=True)
     owner = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, related_name="projects")
     members = models.ManyToManyField(User, related_name="user_projects", blank=True)
+
     feature_flags = SchemaField(
         ProjectFeatureFlags,
         default=default_feature_flags,
         null=False,
         blank=True,
     )
 
+    active = models.BooleanField(default=True)
+    priority = models.IntegerField(default=1)
+
     # Backreferences for type hinting
     captures: models.QuerySet["SourceImage"]
     deployments: models.QuerySet["Deployment"]
     events: models.QuerySet["Event"]
     occurrences: models.QuerySet["Occurrence"]
     taxa: models.QuerySet["Taxon"]
     taxa_lists: models.QuerySet["TaxaList"]
-
-    active = models.BooleanField(default=True)
-    priority = models.IntegerField(default=1)
-
     devices: models.QuerySet["Device"]
     sites: models.QuerySet["Site"]
     jobs: models.QuerySet["Job"]

ami/main/models_future/__init__.py

@@ -0,0 +1,20 @@
+"""
+This is a temporary module for adding new models and features while we migrate models.py to a more modular structure.
+Once the migration is complete, this module will be removed and the models will be moved to their respective files.
+
+This will happen after the current PRs are merged to minimize conflicts.
+
+Current models will be moved to:
+
+models/
+├── __init__.py          # Import everything for backward compatibility
+├── base.py             # BaseModel and mixins
+├── projects.py         # Project, Device, Site, Deployment
+├── storage.py          # S3StorageSource, SourceImageUpload
+├── images.py           # SourceImage, Event, SourceImageCollection
+├── taxonomy.py         # Taxon, TaxaList, Tag
+├── detection.py        # Detection, Classification, Occurrence
+├── identification.py   # Identification
+├── content.py          # Page, BlogPost
+└── enums.py           # TaxonRank and other enums
+"""

ami/main/models_future/projects.py

@@ -0,0 +1,93 @@
+from django.db import models
+
+
+class ProjectSettingsMixin(models.Model):
+    """
+    User definable settings for projects.
+
+    This is a mixin that will be flattened out into the final model.
+    It allows us to organize user-defined project settings in their own class
+    without needing to create a separate model.
+    """
+
+    default_processing_pipeline = models.ForeignKey(
+        "ml.Pipeline",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="default_projects",
+        help_text=(
+            "The default pipeline to use for processing images in this project. "
+            "This is used to determine which processing service to run on new images. "
+            # "Use the global unique key of the Pipeline, which may be a slug or a UUID. "
+            # "Cannot be a Pipeline instance yet, because the Pipelines have likely not been created yet."
+        ),
+    )
+
+    session_time_gap_seconds = models.IntegerField(
+        default=60 * 60 * 2,  # Default to 2 hours
+        help_text="Time gap in seconds to consider a new session",
+    )
+
+    default_filters_score_threshold = models.FloatField(
+        default=0.5,
+        help_text="Default score threshold for filtering occurrences",
+    )
+
+    default_filters_include_taxa = models.ManyToManyField(
+        "Taxon",
+        related_name="include_taxa_default_projects",
+        blank=True,
+        help_text=(
+            "Taxa that are included by default in the occurrence filters and metrics. "
+            "For example, the top-level taxa like 'Moths' or 'Arthropods'. "
+        ),
+    )
+
+    default_filters_exclude_taxa = models.ManyToManyField(
+        "Taxon",
+        related_name="exclude_taxa_default_projects",
+        blank=True,
+        help_text=(
+            "Taxa that are excluded by default in the occurrence filters and metrics. " "For example, 'Not a Moth'."
+        ),
+    )
+
+    # settings_updated_at = models.DateTimeField(auto_now=True, help_text="Last time any setting was updated")
+
+    class Meta:
+        # Do not create a separate table for this mixin
+        abstract = True
+
+    @classmethod
+    def get_settings_field_names(cls) -> list[str]:
+        """
+        Automatically discover settings fields by comparing with BaseModel.
+
+        This finds fields that exist in the final model but not in BaseModel,
+        which means they were added by this mixin.
+        """
+        from ami.base.models import BaseModel  # Adjust import as needed
+
+        # Get all field names from the final model
+        all_fields = {f.name for f in cls._meta.get_fields()}
+
+        # Get all field names from BaseModel
+        base_fields = {f.name for f in BaseModel._meta.get_fields()}
+
+        # Settings fields are those not in BaseModel
+        settings_fields = all_fields - base_fields
+
+        # Filter out reverse relations and other auto-generated fields
+        real_settings_fields = []
+        for field_name in settings_fields:
+            try:
+                field = cls._meta.get_field(field_name)
+                # Skip auto-created fields and foreign key ID fields
+                if not (field.auto_created or field.name.endswith("_id")):
+                    real_settings_fields.append(field_name)
+            except Exception:
+                # Skip fields that can't be retrieved (like reverse relations)
+                continue
+
+        return sorted(real_settings_fields)

ami/users/signals.py

@@ -15,8 +15,17 @@ def create_roles(sender, **kwargs):
     """Creates predefined roles with specific permissions ."""
 
     logger.info("Creating roles for all projects")
-    for project in Project.objects.all():
-        create_roles_for_project(project)
+    try:
+        for project in Project.objects.all():
+            try:
+                create_roles_for_project(project)
+            except Exception as e:
+                logger.warning(f"Failed to create roles for project {project.pk} ({project.name}): {e}")
+                continue
+    except Exception as e:
+        logger.warning(
+            f"Failed to create roles during migration: {e}. This can be run manually via management command."
+        )
 
 
 @receiver(m2m_changed, sender=Group.user_set.through)