Skip to content

Commit 5695c3c

Browse files
mohamedelabbas1996mohamedelabbas1996
authored
Merge branch 'main' into feat/species-of-interest-filter
2 parents aa552a0 + d494761 commit 5695c3c

File tree

16 files changed

+287
-8
lines changed

16 files changed

+287
-8
lines changed

‎ami/jobs/models.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -399,7 +399,9 @@ def run(cls, job: "Job"):
399399
total_detections = 0
400400
total_classifications = 0
401401

402-
CHUNK_SIZE = 4 # Keep it low to see more progress updates
402+
# Set to low size because our response JSON just got enormous
403+
# @TODO make this configurable
404+
CHUNK_SIZE = 1
403405
chunks = [images[i : i + CHUNK_SIZE] for i in range(0, image_count, CHUNK_SIZE)] # noqa
404406
request_failed_images = []
405407

‎ami/main/admin.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import ami.utils
1212
from ami import tasks
13+
from ami.ml.models.project_pipeline_config import ProjectPipelineConfig
1314
from ami.ml.tasks import remove_duplicate_classifications
1415

1516
from .models import (
@@ -30,6 +31,11 @@
3031
)
3132

3233

34+
class ProjectPipelineConfigInline(admin.TabularInline):
35+
model = ProjectPipelineConfig
36+
extra = 0
37+
38+
3339
class AdminBase(admin.ModelAdmin):
3440
"""Mixin to add ``created_at`` and ``updated_at`` to admin panel."""
3541

@@ -65,9 +71,11 @@ def save_related(self, request, form, formsets, change):
6571

6672
list_display = ("name", "owner", "priority", "active", "created_at", "updated_at")
6773
list_filter = ("active", "owner")
68-
search_fields = ("name", "owner__username", "members__username")
74+
search_fields = ("name", "owner__email", "members__email")
6975
filter_horizontal = ("members",)
7076

77+
inlines = [ProjectPipelineConfigInline]
78+
7179
fieldsets = (
7280
(None, {"fields": ("name", "description", "priority", "active")}),
7381
(

‎ami/ml/admin.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from django.contrib import admin
22

3-
from ami.main.admin import AdminBase
3+
from ami.main.admin import AdminBase, ProjectPipelineConfigInline
44

55
from .models.algorithm import Algorithm, AlgorithmCategoryMap
66
from .models.pipeline import Pipeline
@@ -34,6 +34,7 @@ class AlgorithmAdmin(AdminBase):
3434

3535
@admin.register(Pipeline)
3636
class PipelineAdmin(AdminBase):
37+
inlines = [ProjectPipelineConfigInline]
3738
list_display = [
3839
"name",
3940
"version",

‎ami/ml/migrations/0020_projectpipelineconfig_alter_pipeline_projects.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# Generated by Django 4.2.10 on 2025-03-09 23:41
2+
3+
import logging
4+
5+
from django.db import migrations, models
6+
import django.db.models.deletion
7+
from django.utils import timezone
8+
9+
logger = logging.getLogger(__name__)
10+
11+
12+
def transfer_m2m_data(apps, schema_editor):
13+
Pipeline = apps.get_model("ml", "Pipeline")
14+
ProjectPipelineConfig = apps.get_model("ml", "ProjectPipelineConfig")
15+
16+
logger.info("Transferring Project-Pipeline M2M data to ProjectPipelineConfig")
17+
for pipeline in Pipeline.objects.only("id").all():
18+
# Create new ProjectPipelineConfig entries
19+
for existing_project in pipeline.projects.only("id").all():
20+
logger.info(f"Creating ProjectPipelineConfig for {existing_project} and {pipeline}")
21+
ProjectPipelineConfig.objects.create(
22+
project_id=existing_project.pk,
23+
pipeline_id=pipeline.pk,
24+
enabled=True,
25+
config={},
26+
)
27+
28+
29+
def revert_m2m_data(apps, schema_editor):
30+
# collect all projects related to pipelines through ProjectPipelineConfig
31+
# and set the projects field in Pipeline to those projects
32+
ProjectPipelineConfig = apps.get_model("ml", "ProjectPipelineConfig")
33+
Pipeline = apps.get_model("ml", "Pipeline")
34+
Project = apps.get_model("main", "Project")
35+
36+
logger.info("Reverting Project-Pipeline M2M data from ProjectPipelineConfig")
37+
for pipeline in Pipeline.objects.only("id").all():
38+
project_ids = ProjectPipelineConfig.objects.filter(pipeline_id=pipeline.pk).values_list(
39+
"project_id", flat=True
40+
)
41+
projects = Project.objects.filter(id__in=project_ids)
42+
pipeline.projects.set(projects)
43+
44+
45+
# Generated by Django 4.2.10 on 2025-03-09 23:41
46+
47+
48+
class Migration(migrations.Migration):
49+
dependencies = [
50+
("main", "0058_alter_project_options"),
51+
("ml", "0019_alter_algorithm_task_type"),
52+
]
53+
54+
operations = [
55+
migrations.CreateModel(
56+
name="ProjectPipelineConfig",
57+
fields=[
58+
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
59+
("created_at", models.DateTimeField(auto_now_add=True)),
60+
("updated_at", models.DateTimeField(auto_now=True)),
61+
("enabled", models.BooleanField(default=True)),
62+
("config", models.JSONField(blank=True, default=dict, null=True)),
63+
(
64+
"pipeline",
65+
models.ForeignKey(
66+
on_delete=django.db.models.deletion.CASCADE,
67+
related_name="project_pipeline_configs",
68+
to="ml.pipeline",
69+
),
70+
),
71+
(
72+
"project",
73+
models.ForeignKey(
74+
on_delete=django.db.models.deletion.CASCADE,
75+
related_name="project_pipeline_configs",
76+
to="main.project",
77+
),
78+
),
79+
],
80+
options={
81+
"verbose_name": "Project-Pipeline Configuration",
82+
"verbose_name_plural": "Project-Pipeline Configurations",
83+
"unique_together": {("pipeline", "project")},
84+
},
85+
),
86+
migrations.AddField(
87+
model_name="pipeline",
88+
name="new_projects",
89+
field=models.ManyToManyField(
90+
blank=True, related_name="new_pipelines", through="ml.ProjectPipelineConfig", to="main.Project"
91+
),
92+
),
93+
migrations.RunPython(transfer_m2m_data, reverse_code=revert_m2m_data),
94+
migrations.RemoveField(
95+
model_name="pipeline",
96+
name="projects",
97+
),
98+
migrations.RenameField(
99+
model_name="pipeline",
100+
old_name="new_projects",
101+
new_name="projects",
102+
),
103+
migrations.AlterField(
104+
model_name="pipeline",
105+
name="projects",
106+
field=models.ManyToManyField(
107+
blank=True, related_name="pipelines", through="ml.ProjectPipelineConfig", to="main.Project"
108+
),
109+
),
110+
]

‎ami/ml/models/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
from ami.ml.models.algorithm import Algorithm, AlgorithmCategoryMap
22
from ami.ml.models.pipeline import Pipeline
33
from ami.ml.models.processing_service import ProcessingService
4+
from ami.ml.models.project_pipeline_config import ProjectPipelineConfig
45

56
__all__ = [
67
"Algorithm",
78
"AlgorithmCategoryMap",
89
"Pipeline",
910
"ProcessingService",
11+
"ProjectPipelineConfig",
1012
]

‎ami/ml/models/pipeline.py‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from typing import TYPE_CHECKING
44

55
if TYPE_CHECKING:
6-
from ami.ml.models import ProcessingService
6+
from ami.ml.models import ProcessingService # , ProjectPipelineConfig
77

88
import collections
99
import dataclasses
@@ -158,6 +158,7 @@ def process_images(
158158
endpoint_url: str,
159159
images: typing.Iterable[SourceImage],
160160
job_id: int | None = None,
161+
project_id: int | None = None,
161162
) -> PipelineResultsResponse:
162163
"""
163164
Process images using ML pipeline API.
@@ -200,9 +201,34 @@ def process_images(
200201
if url
201202
]
202203

204+
if project_id:
205+
try:
206+
config = pipeline.project_pipeline_configs.get(project_id=project_id).config
207+
task_logger.info(
208+
f"Sending pipeline request using {config} from the project-pipeline config "
209+
f"for Pipeline {pipeline} and Project id {project_id}."
210+
)
211+
except pipeline.project_pipeline_configs.model.DoesNotExist as e:
212+
task_logger.error(
213+
f"Error getting the project-pipeline config for Pipeline {pipeline} "
214+
f"and Project id {project_id}: {e}"
215+
)
216+
config = {}
217+
task_logger.info(
218+
"Using empty config when sending pipeline request since no project-pipeline config "
219+
f"was found for Pipeline {pipeline} and Project id {project_id}"
220+
)
221+
else:
222+
config = {}
223+
task_logger.info(
224+
"Using empty config when sending pipeline request "
225+
f"since no project id was provided for Pipeline {pipeline}"
226+
)
227+
203228
request_data = PipelineRequest(
204229
pipeline=pipeline.slug,
205230
source_images=source_images,
231+
config=config,
206232
)
207233

208234
session = create_session()
@@ -897,8 +923,11 @@ class Pipeline(BaseModel):
897923
"The backend implementation of the pipeline may process data in any way."
898924
),
899925
)
900-
projects = models.ManyToManyField("main.Project", related_name="pipelines", blank=True)
926+
projects = models.ManyToManyField(
927+
"main.Project", related_name="pipelines", blank=True, through="ml.ProjectPipelineConfig"
928+
)
901929
processing_services: models.QuerySet[ProcessingService]
930+
# project_pipeline_configs: models.QuerySet[ProjectPipelineConfig]
902931

903932
class Meta:
904933
ordering = ["name", "version"]
@@ -988,6 +1017,7 @@ def process_images(self, images: typing.Iterable[SourceImage], project_id: int,
9881017
pipeline=self,
9891018
images=images,
9901019
job_id=job_id,
1020+
project_id=project_id,
9911021
)
9921022

9931023
def save_results(self, results: PipelineResultsResponse, job_id: int | None = None):

‎ami/ml/models/processing_service.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
from ami.base.models import BaseModel
1111
from ami.ml.models.pipeline import Pipeline, get_or_create_algorithm_and_category_map
12+
from ami.ml.models.project_pipeline_config import ProjectPipelineConfig
1213
from ami.ml.schemas import PipelineRegistrationResponse, ProcessingServiceInfoResponse, ProcessingServiceStatusResponse
1314

1415
logger = logging.getLogger(__name__)
@@ -59,7 +60,18 @@ def create_pipelines(self):
5960
)
6061
created = True
6162

62-
pipeline.projects.add(*self.projects.all())
63+
for project in self.projects.all():
64+
project_pipeline_config, created = ProjectPipelineConfig.objects.get_or_create(
65+
pipeline=pipeline,
66+
project=project,
67+
defaults={"enabled": True, "config": {}},
68+
)
69+
if created:
70+
logger.info(f"Created project pipeline config for {project.name} and {pipeline.name}.")
71+
project_pipeline_config.save()
72+
else:
73+
logger.info(f"Using existing project pipeline config for {project.name} and {pipeline.name}.")
74+
6375
self.pipelines.add(pipeline)
6476

6577
if created:

‎ami/ml/models/project_pipeline_config.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import logging
2+
import typing
3+
4+
from django.db import models
5+
6+
from ami.base.models import BaseModel
7+
8+
# from ami.main.models import Project
9+
# from ami.ml.models.pipeline import Pipeline
10+
11+
logger = logging.getLogger(__name__)
12+
13+
14+
@typing.final
15+
class ProjectPipelineConfig(BaseModel):
16+
"""Intermediate model to store the relationship between a project and a pipeline."""
17+
18+
project = models.ForeignKey("main.Project", related_name="project_pipeline_configs", on_delete=models.CASCADE)
19+
pipeline = models.ForeignKey("ml.Pipeline", related_name="project_pipeline_configs", on_delete=models.CASCADE)
20+
enabled = models.BooleanField(default=True)
21+
config = models.JSONField(default=dict, blank=True, null=True)
22+
23+
def __str__(self):
24+
return f'#{self.pk} "{self.pipeline}" in {self.project}'
25+
26+
class Meta:
27+
unique_together = ("pipeline", "project")
28+
verbose_name = "Project-Pipeline Configuration"
29+
verbose_name_plural = "Project-Pipeline Configurations"

‎ami/ml/schemas.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ class Config:
147147
class PipelineRequest(pydantic.BaseModel):
148148
pipeline: str
149149
source_images: list[SourceImageRequest]
150+
config: dict
150151

151152

152153
class PipelineResultsResponse(pydantic.BaseModel):

‎ami/ml/serializers.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from .models.algorithm import Algorithm, AlgorithmCategoryMap
88
from .models.pipeline import Pipeline, PipelineStage
99
from .models.processing_service import ProcessingService
10+
from .models.project_pipeline_config import ProjectPipelineConfig
1011

1112

1213
class AlgorithmCategoryMapSerializer(DefaultSerializer):
@@ -73,10 +74,23 @@ class Meta:
7374
]
7475

7576

77+
class ProjectPipelineConfigSerializer(DefaultSerializer):
78+
class Meta:
79+
model = ProjectPipelineConfig
80+
fields = [
81+
"id",
82+
"project",
83+
"pipeline",
84+
"enabled",
85+
"config",
86+
]
87+
88+
7689
class PipelineSerializer(DefaultSerializer):
7790
algorithms = AlgorithmSerializer(many=True, read_only=True)
7891
stages = SchemaField(schema=list[PipelineStage], read_only=True)
7992
processing_services = ProcessingServiceNestedSerializer(many=True, read_only=True)
93+
project_pipeline_configs = ProjectPipelineConfigSerializer(many=True, read_only=True)
8094

8195
class Meta:
8296
model = Pipeline
@@ -89,6 +103,7 @@ class Meta:
89103
"algorithms",
90104
"stages",
91105
"processing_services",
106+
"project_pipeline_configs",
92107
"created_at",
93108
"updated_at",
94109
]

0 commit comments

Comments
 (0)