Skip to content

Commit 8fa0d32

Browse files
maxcorbeaumaxcorbeau
committed
rebase + ajout migration & tests
1 parent a9c9ee7 commit 8fa0d32

File tree

9 files changed

+134
-25
lines changed

9 files changed

+134
-25
lines changed

‎dags/enrich/config/dbt.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@ class DBT:
1919
MARTS_ENRICH_AE_CLOSED_NOT_REPLACED: str = (
2020
"marts_enrich_acteurs_closed_suggest_not_replaced"
2121
)
22-
MARTS_ENRICH_ACTEURS_VILLES_TYPO: str = "marts_enrich_acteurs_villes_suggest_typo"
23-
MARTS_ENRICH_ACTEURS_VILLES_NEW: str = "marts_enrich_acteurs_villes_suggest_new"
22+
MARTS_ENRICH_VILLES_TYPO: str = "marts_enrich_acteurs_villes_suggest_typo"
23+
MARTS_ENRICH_VILLES_NEW: str = "marts_enrich_acteurs_villes_suggest_new"

‎dags/enrich/config/tasks.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,5 @@ class TASKS:
2929
ENRICH_DBT_MODELS_REFRESH: str = "enrich_dbt_models_refresh"
3030

3131
# Villes
32-
ENRICH_ACTEURS_VILLES_TYPO: str = "enrich_acteurs_villes_typo"
33-
ENRICH_ACTEURS_VILLES_NEW: str = "enrich_acteurs_villes_new"
32+
ENRICH_VILLES_TYPO: str = "enrich_acteurs_villes_typo"
33+
ENRICH_VILLES_NEW: str = "enrich_acteurs_villes_new"

‎dags/enrich/dags/enrich_acteurs_villes.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,15 @@
4747
dbt_refresh = enrich_dbt_models_refresh_task(dag)
4848
suggest_typo = enrich_dbt_model_suggest_task(
4949
dag,
50-
task_id=TASKS.ENRICH_ACTEURS_VILLES_TYPO,
51-
cohort=COHORTS.ACTEURS_VILLES_TYPO,
52-
dbt_model_name=DBT.MARTS_ENRICH_ACTEURS_VILLES_TYPO,
50+
task_id=TASKS.ENRICH_VILLES_TYPO,
51+
cohort=COHORTS.VILLES_TYPO,
52+
dbt_model_name=DBT.MARTS_ENRICH_VILLES_TYPO,
5353
)
5454
suggest_new = enrich_dbt_model_suggest_task(
5555
dag,
56-
task_id=TASKS.ENRICH_ACTEURS_VILLES_NEW,
57-
cohort=COHORTS.ACTEURS_VILLES_NEW,
58-
dbt_model_name=DBT.MARTS_ENRICH_ACTEURS_VILLES_NEW,
56+
task_id=TASKS.ENRICH_VILLES_NEW,
57+
cohort=COHORTS.VILLES_NEW,
58+
dbt_model_name=DBT.MARTS_ENRICH_VILLES_NEW,
5959
)
6060
config >> dbt_refresh # type: ignore
6161
dbt_refresh >> suggest_typo # type: ignore

‎dags/enrich/tasks/business_logic/enrich_dbt_model_to_suggestions.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def changes_prepare_rgpd(
6161
return changes, contexte
6262

6363

64-
def changes_prepare_villes(row: dict) -> list[dict]:
64+
def changes_prepare_villes(row: dict) -> tuple[list[dict], dict]:
6565
"""Prepare suggestions for villes cohorts"""
6666
from data.models.changes import ChangeActeurUpdateData
6767

@@ -81,7 +81,8 @@ def changes_prepare_villes(row: dict) -> list[dict]:
8181
entity_type="acteur_displayed",
8282
)
8383
)
84-
return changes
84+
contexte = {} # changes are self-explanatory
85+
return changes, contexte
8586

8687

8788
def changes_prepare_closed_not_replaced(
@@ -207,8 +208,8 @@ def changes_prepare_closed_replaced(
207208
COHORTS.CLOSED_REP_OTHER_SIREN: changes_prepare_closed_replaced,
208209
COHORTS.CLOSED_REP_SAME_SIREN: changes_prepare_closed_replaced,
209210
COHORTS.RGPD: changes_prepare_rgpd,
210-
COHORTS.ACTEURS_VILLES_TYPO: changes_prepare_villes,
211-
COHORTS.ACTEURS_VILLES_NEW: changes_prepare_villes,
211+
COHORTS.VILLES_TYPO: changes_prepare_villes,
212+
COHORTS.VILLES_NEW: changes_prepare_villes,
212213
}
213214

214215

@@ -225,12 +226,16 @@ def enrich_dbt_model_to_suggestions(
225226
SuggestionStatut,
226227
)
227228

229+
# TODO: once all suggestions have been migrated to pydantic, we no
230+
# longer need SuggestionCohorte.type_action and any of the following
231+
# identifiant_execution = cohort AND pydantic models take care of
232+
# handling the specifics
228233
COHORTS_TO_SUGGESTION_ACTION = {
229234
COHORTS.CLOSED_NOT_REPLACED: SuggestionAction.ENRICH_ACTEURS_CLOSED,
230235
COHORTS.CLOSED_REP_OTHER_SIREN: SuggestionAction.ENRICH_ACTEURS_CLOSED,
231236
COHORTS.CLOSED_REP_SAME_SIREN: SuggestionAction.ENRICH_ACTEURS_CLOSED,
232-
COHORTS.ACTEURS_VILLES_TYPO: SuggestionAction.ENRICH_ACTEURS_VILLES_TYPO,
233-
COHORTS.ACTEURS_VILLES_NEW: SuggestionAction.ENRICH_ACTEURS_VILLES_NEW,
237+
COHORTS.VILLES_TYPO: SuggestionAction.ENRICH_ACTEURS_VILLES_TYPO,
238+
COHORTS.VILLES_NEW: SuggestionAction.ENRICH_ACTEURS_VILLES_NEW,
234239
}
235240

236241
# Validation

‎dags_unit_tests/enrich/tasks/test_enrich_suggestions_cities.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import pandas as pd
2+
import pytest
3+
from enrich.config import COHORTS, COLS
4+
from enrich.tasks.business_logic.enrich_dbt_model_to_suggestions import (
5+
enrich_dbt_model_to_suggestions,
6+
)
7+
8+
9+
@pytest.mark.django_db
10+
class TestEnrichSuggestionsCities:
11+
12+
@pytest.fixture
13+
def df_new(self):
14+
return pd.DataFrame(
15+
{
16+
COLS.SUGGEST_COHORT: [COHORTS.VILLES_NEW] * 2,
17+
COLS.SUGGEST_VILLE: ["new town 1", "new town 2"],
18+
COLS.ACTEUR_ID: ["new1", "new2"],
19+
COLS.ACTEUR_VILLE: ["old town 1", "old town 2"],
20+
}
21+
)
22+
23+
@pytest.fixture
24+
def df_typo(self):
25+
return pd.DataFrame(
26+
{
27+
COLS.SUGGEST_COHORT: [COHORTS.VILLES_TYPO] * 2,
28+
COLS.SUGGEST_VILLE: ["Paris", "Laval"],
29+
COLS.ACTEUR_ID: ["typo1", "typo2"],
30+
COLS.ACTEUR_VILLE: ["Pâris", "Lâval"],
31+
}
32+
)
33+
34+
@pytest.fixture
35+
def acteurs(self, df_new, df_typo):
36+
# Creating acteurs as presence required to apply changes
37+
from unit_tests.qfdmo.acteur_factory import ActeurFactory
38+
39+
for _, row in pd.concat([df_new, df_typo]).iterrows():
40+
ActeurFactory(
41+
identifiant_unique=row[COLS.ACTEUR_ID],
42+
ville=row[COLS.ACTEUR_VILLE],
43+
)
44+
45+
def test_cohort_new(self, acteurs, df_new):
46+
from data.models.suggestion import Suggestion, SuggestionCohorte
47+
from qfdmo.models import RevisionActeur
48+
49+
# Write suggestions to DB
50+
enrich_dbt_model_to_suggestions(
51+
df=df_new,
52+
cohort=COHORTS.VILLES_NEW,
53+
identifiant_action="test_new",
54+
dry_run=False,
55+
)
56+
57+
# Check suggestions have been written to DB
58+
cohort = SuggestionCohorte.objects.get(identifiant_action="test_new")
59+
suggestions = Suggestion.objects.filter(suggestion_cohorte=cohort)
60+
assert len(suggestions) == 2
61+
62+
# Apply suggestions
63+
for suggestion in suggestions:
64+
suggestion.apply()
65+
66+
# Verify changes
67+
# 2 revisions should be created but not parent
68+
new1 = RevisionActeur.objects.get(pk="new1")
69+
assert new1.ville == "new town 1"
70+
71+
new2 = RevisionActeur.objects.get(pk="new2")
72+
assert new2.ville == "new town 2"
73+
74+
def test_cohort_typo(self, acteurs, df_typo):
75+
from data.models.suggestion import Suggestion, SuggestionCohorte
76+
from qfdmo.models import RevisionActeur
77+
78+
# Write suggestions to DB
79+
enrich_dbt_model_to_suggestions(
80+
df=df_typo,
81+
cohort=COHORTS.VILLES_TYPO,
82+
identifiant_action="test_typo",
83+
dry_run=False,
84+
)
85+
86+
# Check suggestions have been written to DB
87+
cohort = SuggestionCohorte.objects.get(identifiant_action="test_typo")
88+
suggestions = Suggestion.objects.filter(suggestion_cohorte=cohort)
89+
assert len(suggestions) == 2
90+
91+
# Apply suggestions
92+
for suggestion in suggestions:
93+
suggestion.apply()
94+
95+
# Verify changes
96+
typo1 = RevisionActeur.objects.get(pk="typo1")
97+
assert typo1.ville == "Paris"
98+
99+
typo2 = RevisionActeur.objects.get(pk="typo2")
100+
assert typo2.ville == "Laval"

‎data/models/suggestion.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ def display_contexte_details(self):
189189
SuggestionAction.CLUSTERING,
190190
SuggestionAction.CRAWL_URLS,
191191
SuggestionAction.ENRICH_ACTEURS_RGPD,
192+
SuggestionAction.ENRICH_ACTEURS_VILLES_TYPO,
193+
SuggestionAction.ENRICH_ACTEURS_VILLES_NEW,
192194
]:
193195
context["details_open"] = True
194196

@@ -210,6 +212,8 @@ def display_suggestion_details(self):
210212
elif self.suggestion_cohorte.type_action in [
211213
SuggestionAction.ENRICH_ACTEURS_CLOSED,
212214
SuggestionAction.ENRICH_ACTEURS_RGPD,
215+
SuggestionAction.ENRICH_ACTEURS_VILLES_TYPO,
216+
SuggestionAction.ENRICH_ACTEURS_VILLES_NEW,
213217
]:
214218
template_name = "data/_partials/suggestion_details_changes.html"
215219
template_context = self.suggestion
@@ -334,6 +338,8 @@ def apply(self):
334338
SuggestionAction.CRAWL_URLS,
335339
SuggestionAction.ENRICH_ACTEURS_CLOSED,
336340
SuggestionAction.ENRICH_ACTEURS_RGPD,
341+
SuggestionAction.ENRICH_ACTEURS_VILLES_TYPO,
342+
SuggestionAction.ENRICH_ACTEURS_VILLES_NEW,
337343
]:
338344
changes = self.suggestion["changes"]
339345
changes.sort(key=lambda x: x["order"])

‎dbt/models/marts/enrich/marts_enrich_acteurs_villes_suggest.sql‎ renamed to ‎dbt/models/marts/enrich/marts_enrich_acteurs_villes_candidates.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ SELECT
1212
ban.ville_ancienne AS ban_ville_ancienne,
1313
ban.ville AS ban_ville,
1414
ban.code_postal AS ban_code_postal,
15-
ban.ville AS remplacer_ville
15+
ban.ville AS suggest_ville
1616
FROM {{ ref('marts_carte_acteur') }} AS acteurs
1717
JOIN {{ ref('int_ban_villes') }} AS ban ON ban.code_postal = acteurs.code_postal
1818
WHERE acteurs.statut = 'ACTIF'

‎dbt/models/marts/enrich/marts_enrich_acteurs_villes_suggest_new.sql‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
}}
88

99
SELECT
10-
'acteurs_villes_anciennes_nouvelles' AS suggestion_cohorte_code,
11-
'🌆 Changement de ville: 🟡 ancienne -> nouvelle' AS suggestion_cohorte_label,
10+
'🌆 Changement de ville: 🟡 ancienne -> nouvelle' AS suggest_cohort,
1211
*
13-
FROM {{ ref('marts_enrich_acteurs_villes_suggest') }}
14-
WHERE udf_normalize_string_for_match(acteur_ville,3) != udf_normalize_string_for_match(remplacer_ville,3)
12+
FROM {{ ref('marts_enrich_acteurs_villes_candidates') }}
13+
WHERE udf_normalize_string_for_match(acteur_ville,3) != udf_normalize_string_for_match(suggest_ville,3)
1514
AND ban_ville_ancienne IS NOT NULL

‎dbt/models/marts/enrich/marts_enrich_acteurs_villes_suggest_typo.sql‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
}}
88

99
SELECT
10-
'acteurs_villes_variation_ortographe' AS suggestion_cohorte_code,
11-
'🌆 Changement de ville: 🟢 variation d''ortographe' AS suggestion_cohorte_label,
10+
'🌆 Changement de ville: 🟢 variation d''ortographe' AS suggest_cohort,
1211
*
13-
FROM {{ ref('marts_enrich_acteurs_villes_suggest') }}
14-
WHERE udf_normalize_string_for_match(acteur_ville,3) = udf_normalize_string_for_match(remplacer_ville,3)
12+
FROM {{ ref('marts_enrich_acteurs_villes_candidates') }}
13+
WHERE udf_normalize_string_for_match(acteur_ville,3) = udf_normalize_string_for_match(suggest_ville,3)
1514
AND ban_ville_ancienne IS NULL

0 commit comments

Comments
 (0)