Merge branch 'main' into multi-sous-categories

Author: fabienheureux

.github/workflows/review.yml

@@ -3,7 +3,7 @@ name: 💣 Review
 on:
   workflow_dispatch:
   pull_request:
-    types: [labeled, opened, synchronize]
+    types: [opened, synchronize]
 
 defaults:
   run:
@@ -14,9 +14,17 @@ jobs:
     name: 🤖 CI
     uses: ./.github/workflows/ci.yml
 
-  preprod:
-    name: 🟠 Preprod
-    uses: ./.github/workflows/_cd-by-environment.yml
+  webapp:
+    name: 🟠 Webapp
+    uses: ./.github/workflows/_deploy-webapp.yml
+    secrets: inherit # pragma: allowlist secret`
+    needs: [ci]
+    with:
+      environment: preprod
+
+  airflow:
+    name: 🟠 Airflow
+    uses: ./.github/workflows/_deploy-airflow.yml
     secrets: inherit # pragma: allowlist secret`
     needs: [ci]
     with:

.github/workflows/terragrunt_lint.yml

@@ -0,0 +1,26 @@
+name: 'Terragrunt Lint'
+on:
+  - pull_request
+
+jobs:
+  terragrunt:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Terragrunt and OpenTofu
+        uses: gruntwork-io/terragrunt-action@95fc057922e3c3d4cc021a81a213f088f333ddef
+        with:
+          tg_version: '0.83.2'
+          tofu_version: '1.10.3'
+
+      - name: Run Tofu format check
+        run: |
+          cd infrastructure/
+          tofu fmt -recursive -check
+
+# Address actions/missing-workflow-permissions rule in
+# code scanning alerts
+permissions:
+  contents: read

.secrets.baseline

@@ -160,7 +160,7 @@
         "filename": "docker-compose.yml",
         "hashed_secret": "3cf2012487b086bba2adb3386d69c2ab67a268b6",
         "is_verified": false,
-        "line_number": 57
+        "line_number": 58
       }
     ],
     "iframe_without_js.html": [
@@ -191,5 +191,5 @@
       }
     ]
   },
-  "generated_at": "2025-07-17T08:29:41Z"
+  "generated_at": "2025-08-01T12:06:26Z"
 }

airflow-scheduler.Dockerfile

@@ -28,10 +28,13 @@ USER root
 # unzip for Airflow DAG
 RUN echo "deb http://deb.debian.org/debian stable main" > /etc/apt/sources.list
 RUN apt-get update
-RUN apt-get install -y unzip
+RUN apt-get install -y unzip curl
 
 RUN apt-get install -y --no-install-recommends \
-    gdal-bin libgdal-dev
+    gdal-bin libgdal-dev jq
+
+# Installation du client Scaleway CLI
+RUN curl -s https://raw.githubusercontent.com/scaleway/scaleway-cli/master/scripts/get.sh | sh
 
 USER ${AIRFLOW_UID:-50000}:0
 WORKDIR /opt/airflow
@@ -47,6 +50,7 @@ COPY ./qfdmo/ /opt/airflow/qfdmo/
 COPY ./qfdmd/ /opt/airflow/qfdmd/
 COPY ./data/ /opt/airflow/data/
 COPY ./dbt/ /opt/airflow/dbt/
+COPY ./scripts/ /opt/airflow/scripts/
 COPY ./dsfr_hacks/ /opt/airflow/dsfr_hacks/
 
 # Classique Airflow

dags/.env.template

@@ -62,3 +62,9 @@ POSTGRES_USER=qfdmo
 POSTGRES_PASSWORD=qfdmo
 POSTGRES_DB=warehouse
 POSTGRES_SCHEMA=public
+
+# SCALEWAY
+SCW_ACCESS_KEY=
+SCW_SECRET_KEY=
+SCW_DEFAULT_ORGANIZATION_ID=
+SCW_DEFAULT_PROJECT_ID=

dags/cluster/dags/cluster_acteur_suggestions.py

@@ -273,15 +273,16 @@
         type="boolean",
         description_md=r"""**∅ CONSERVER LE VIDE**: si OUI et qu'une valeur
         vide est rencontrée sur une source prioritaire, alors elle sera
-        conservée""",
+        conservée.
+        **Cette option n'est appliquée que lors de la mis à jour du parent**""",
     ),
     "dedup_enrich_keep_parent_data_by_default": Param(
         True,
         type="boolean",
         description_md=r"""
 ** CONSERVER LES DONNÉES DU PARENT**: si OUI, les données du parent seront conservées.
 
-Lorsque l'option `dedup_enrich_keep_empty` est:
+Dans le cas de la mise à jour du parent, lorsque l'option `dedup_enrich_keep_empty` est:
  - VRAI, toutes les données du parent même vides sont conservées
  - FAUX, seules les données non-vides du parent sont conservées
 """,

dags/cluster/tasks/business_logic/cluster_acteurs_parents_choose_data.py

@@ -113,6 +113,9 @@ def source_priority(a):
         # Acteurs to consider: first revisions, then base, but not from excluded sources
         acteurs = list(acteurs_revision) + list(acteurs_base)
         acteurs.sort(key=source_priority)
+        # On parent creation, we don't want to keep empty data
+        if not parent:
+            keep_empty = False
         if parent and keep_parent_data_by_default:
             acteurs = [parent] + acteurs
 

dags/shared/config/schedules.py

@@ -5,6 +5,7 @@
 
 @dataclass(frozen=True)
 class SCHEDULES:
+    HOURLY: str = "3 * * * *"  # 3 minutes après chaque heure
     DAILY: str = "0 0 * * *"
     DAILY_AT_1AM: str = "0 1 * * *"
     WEEKLY: str = "0 0 * * 1"

dags/shared/config/tags.py

@@ -90,3 +90,4 @@ class TAGS:
     ACTEURS: str = "acteurs"
     TOUT: str = "tout"
     SQL: str = "sql"
+    SCALEWAY: str = "scaleway"

dags/sources/dags/source_soren.py

@@ -27,19 +27,12 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "nom_de_lorganisme",
-                "destination": "nom",
-            },
-            {
-                "origin": "longitudewgs84",
-                "destination": "longitude",
-            },
-            {
-                "origin": "latitudewgs84",
-                "destination": "latitude",
-            },
+            {"origin": "nom_de_lorganisme", "destination": "nom"},
+            {"origin": "enseigne_commerciale", "destination": "nom_commercial"},
+            {"origin": "longitudewgs84", "destination": "longitude"},
+            {"origin": "latitudewgs84", "destination": "latitude"},
             # 2. Transformation des colonnes
+            {"origin": "site_web", "transformation": "clean_url", "destination": "url"},
             {
                 "origin": "ecoorganisme",
                 "transformation": "strip_lower_string",
@@ -86,10 +79,7 @@
                 "destination": "sous_categorie_codes",
             },
             # 3. Ajout des colonnes avec une valeur par défaut
-            {
-                "column": "statut",
-                "value": constants.ACTEUR_ACTIF,
-            },
+            {"column": "statut", "value": constants.ACTEUR_ACTIF},
             # 4. Transformation du dataframe
             {
                 "origin": ["siret", "siren"],
@@ -165,8 +155,10 @@
             {"remove": "point_de_reparation"},
             {"remove": "perimetre_dintervention"},
             {"remove": "service_a_domicile"},
+            {"remove": "consignes_dacces"},
             # 6. Colonnes à garder (rien à faire, utilisé pour le controle)
             {"keep": "email"},
+            {"keep": "adresse_complement"},
         ],
         "endpoint": (
             "https://data.pointsapport.ademe.fr/data-fair/api/v1/datasets/"