Skip to content

Commit b54d356

Browse files
fabienheureuxfabienheureux
authored
Merge branch 'main' into assistant-v2-responsive
2 parents 6646e5b + 7ade97e commit b54d356

28 files changed

+266
-132
lines changed

‎dags/create_final_actors.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ def write_data_to_postgres(**kwargs):
261261
"telephone",
262262
"nom_commercial",
263263
"nom_officiel",
264+
"siren",
264265
"siret",
265266
"identifiant_externe",
266267
"acteur_type_id",

‎dags/sources/config/airflow_params.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@
22
from pathlib import Path
33

44
import requests
5-
from sources.tasks.transform.transform_column import convert_opening_hours
5+
from sources.tasks.transform.transform_column import (
6+
clean_siren,
7+
clean_siret,
8+
convert_opening_hours,
9+
)
610

711
PATH_NOMENCLARURE_DECHET = (
812
"https://data.ademe.fr/data-fair/api/v1/datasets/sinoe-r-nomenclature-dechets/lines"
@@ -13,7 +17,11 @@
1317
KEY_LIBELLE_DECHET_ALT = "LST_TYP_DECHET"
1418

1519

16-
TRANSFORMATION_MAPPING = {"convert_opening_hours": convert_opening_hours}
20+
TRANSFORMATION_MAPPING = {
21+
"convert_opening_hours": convert_opening_hours,
22+
"clean_siren": clean_siren,
23+
"clean_siret": clean_siret,
24+
}
1725

1826

1927
# TODO: dataclass à implémenter pour la validation des paramètres des DAGs

‎dags/sources/dags/source_aliapur.py‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,18 @@
1111
" for Aliapur dataset"
1212
),
1313
params={
14+
"column_transformations": [
15+
{
16+
"origin": "siret",
17+
"transformation": "clean_siret",
18+
"destination": "siret",
19+
},
20+
],
1421
"column_mapping": {
1522
"id_point_apport_ou_reparation": "identifiant_externe",
1623
"type_de_point_de_collecte": "acteur_type_id",
17-
"exclusivite_de_reprisereparation": "exclusivite_de_reprisereparation",
18-
"uniquement_sur_rdv": "uniquement_sur_rdv",
19-
"public_accueilli": "public_accueilli",
20-
"reprise": "reprise",
21-
"produitsdechets_acceptes": "produitsdechets_acceptes",
22-
"labels_etou_bonus": "labels_etou_bonus",
23-
"point_de_reparation": "point_de_reparation",
2424
"ecoorganisme": "source_id",
25-
"adresse_format_ban": "adresse_format_ban",
2625
"nom_de_lorganisme": "nom",
27-
"perimetre_dintervention": "perimetre_dintervention",
2826
"longitudewgs84": "longitude",
2927
"latitudewgs84": "latitude",
3028
},

‎dags/sources/dags/source_citeo.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@
1111
" sur de Koumoul"
1212
),
1313
params={
14+
"column_transformations": [
15+
{
16+
"origin": "siren",
17+
"transformation": "clean_siren",
18+
"destination": "siren",
19+
},
20+
],
1421
"column_mapping": {
1522
"id_point_apport_ou_reparation": "identifiant_externe",
1623
"type_de_point_de_collecte": "acteur_type_id",

‎dags/sources/dags/source_cma.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@
1212
" for CMA reparacteur dataset"
1313
),
1414
params={
15+
"column_transformations": [
16+
{
17+
"origin": "siret",
18+
"transformation": "clean_siret",
19+
"destination": "siret",
20+
},
21+
],
1522
"column_mapping": {
1623
"name": "nom",
1724
"reparactor_description": "description",

‎dags/sources/dags/source_corepile.py‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,19 @@
1111
" for Corepile dataset"
1212
),
1313
params={
14+
"column_transformations": [
15+
{
16+
"origin": "siret",
17+
"transformation": "clean_siret",
18+
"destination": "siret",
19+
},
20+
],
1421
"column_mapping": {
1522
"id_point_apport_ou_reparation": "identifiant_externe",
1623
"type_de_point_de_collecte": "acteur_type_id",
17-
"siret": "siret",
18-
"exclusivite_de_reprisereparation": "exclusivite_de_reprisereparation",
19-
"uniquement_sur_rdv": "uniquement_sur_rdv",
20-
"public_accueilli": "public_accueilli",
21-
"reprise": "reprise",
22-
"produitsdechets_acceptes": "produitsdechets_acceptes",
23-
"labels_etou_bonus": "labels_etou_bonus",
24-
"point_de_reparation": "point_de_reparation",
2524
"ecoorganisme": "source_id",
26-
"adresse_format_ban": "adresse_format_ban",
2725
"nom_de_lorganisme": "nom",
2826
"enseigne_commerciale": "nom_commercial",
29-
"perimetre_dintervention": "perimetre_dintervention",
3027
"longitudewgs84": "longitude",
3128
"latitudewgs84": "latitude",
3229
},

‎dags/sources/dags/source_ecodds.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,6 @@
1515
"id_point_apport_ou_reparation": "identifiant_externe",
1616
"type_de_point_de_collecte": "acteur_type_id",
1717
"ecoorganisme": "source_id",
18-
"adresse_format_ban": "adresse_format_ban",
19-
"exclusivite_de_reprisereparation": "exclusivite_de_reprisereparation",
20-
"uniquement_sur_rdv": "uniquement_sur_rdv",
21-
"public_accueilli": "public_accueilli",
22-
"reprise": "reprise",
2318
"nom_de_lorganisme": "nom",
2419
"enseigne_commerciale": "nom_commercial",
2520
"longitudewgs84": "longitude",

‎dags/sources/dags/source_ecologic.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,8 @@
1414
"column_mapping": {
1515
"id_point_apport_ou_reparation": "identifiant_externe",
1616
"type_de_point_de_collecte": "acteur_type_id",
17-
"exclusivite_de_reprisereparation": "exclusivite_de_reprisereparation",
18-
"uniquement_sur_rdv": "uniquement_sur_rdv",
19-
"public_accueilli": "public_accueilli",
20-
"reprise": "reprise",
21-
"produitsdechets_acceptes": "produitsdechets_acceptes",
22-
"labels_etou_bonus": "labels_etou_bonus",
23-
"point_de_reparation": "point_de_reparation",
2417
"ecoorganisme": "source_id",
25-
"adresse_format_ban": "adresse_format_ban",
2618
"nom_de_lorganisme": "nom",
27-
"perimetre_dintervention": "perimetre_dintervention",
2819
"longitudewgs84": "longitude",
2920
"latitudewgs84": "latitude",
3021
},

‎dags/sources/dags/source_ecomaison.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,25 +11,20 @@
1111
" for Ecomaison dataset"
1212
),
1313
params={
14+
"column_transformations": [
15+
{
16+
"origin": "siret",
17+
"transformation": "clean_siret",
18+
"destination": "siret",
19+
},
20+
],
1421
"column_mapping": {
1522
"id_point_apport_ou_reparation": "identifiant_externe",
1623
"type_de_point_de_collecte": "acteur_type_id",
17-
"exclusivite_de_reprisereparation": "exclusivite_de_reprisereparation",
18-
"uniquement_sur_rdv": "uniquement_sur_rdv",
19-
"public_accueilli": "public_accueilli",
20-
"reprise": "reprise",
2124
"enseigne_commerciale": "nom_commercial",
22-
"telephone": "telephone",
23-
"email": "email",
24-
"siret": "siret",
25-
"produitsdechets_acceptes": "produitsdechets_acceptes",
26-
"labels_etou_bonus": "labels_etou_bonus",
27-
"point_de_reparation": "point_de_reparation",
2825
"ecoorganisme": "source_id",
2926
"site_web": "url",
30-
"adresse_format_ban": "adresse_format_ban",
3127
"nom_de_lorganisme": "nom",
32-
"perimetre_dintervention": "perimetre_dintervention",
3328
"longitudewgs84": "longitude",
3429
"latitudewgs84": "latitude",
3530
},

‎dags/sources/dags/source_ecosystem.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,18 @@
1111
" for Ecosystem dataset"
1212
),
1313
params={
14+
"column_transformations": [
15+
{
16+
"origin": "siren",
17+
"transformation": "clean_siren",
18+
"destination": "siren",
19+
},
20+
{
21+
"origin": "siret",
22+
"transformation": "clean_siret",
23+
"destination": "siret",
24+
},
25+
],
1426
"column_mapping": {
1527
"id_point_apport_ou_reparation": "identifiant_externe",
1628
"type_de_point_de_collecte": "acteur_type_id",

0 commit comments

Comments
 (0)