Preuve de concept de l'utilisation de DBT

Author: kolok

.secrets.baseline

@@ -160,7 +160,7 @@
         "filename": "docker-compose.yml",
         "hashed_secret": "3cf2012487b086bba2adb3386d69c2ab67a268b6",
         "is_verified": false,
-        "line_number": 54
+        "line_number": 55
       }
     ],
     "iframe_without_js.html": [
@@ -207,5 +207,5 @@
       }
     ]
   },
-  "generated_at": "2025-01-29T18:47:34Z"
+  "generated_at": "2025-02-14T10:44:30Z"
 }

airflow-requirements.in

@@ -1,13 +1,15 @@
 --constraint=requirements.txt
 
-apache-airflow==2.10.4
 apache-airflow-providers-postgres
+apache-airflow==2.10.4
+dbt-core==1.9.2
+dbt-postgres==1.9
 fuzzywuzzy
 # related to https://github.com/pandas-dev/pandas/issues/57049 because sqlalchemy & numpy should be < 2.0
 pandas==2.1.4
 pyproj
 python-decouple
 ratelimit
+scikit-learn==1.3.2
 shortuuid
-unidecode
-scikit-learn==1.3.2
+unidecode

airflow-requirements.txt

@@ -35,12 +35,24 @@ COPY ./core/ /opt/airflow/core/
 COPY ./qfdmo/ /opt/airflow/qfdmo/
 COPY ./qfdmd/ /opt/airflow/qfdmd/
 COPY ./data/ /opt/airflow/data/
+COPY ./dbt/ /opt/airflow/dbt/
 COPY ./dsfr_hacks/ /opt/airflow/dsfr_hacks/
 
 # Classique Airflow
 COPY ./dags/ /opt/airflow/dags/
 COPY ./config/ /opt/airflow/config/
 COPY ./plugins/ /opt/airflow/plugins/
-RUN mkdir -p /opt/airflow/logs/
+
+WORKDIR /opt/airflow/dbt
+USER 0
+RUN chown -R ${AIRFLOW_UID:-50000}:0 /opt/airflow/dbt
+USER ${AIRFLOW_UID:-50000}:0
+
+# RUN mkdir -p /opt/airflow/.dbt/logs
+# ENV DBT_LOG_PATH=/opt/airflow/.dbt/logs/dbt.log
+ENV DBT_PROFILES_DIR=/opt/airflow/dbt
+ENV DBT_PROJECT_DIR=/opt/airflow/dbt
+
+RUN dbt deps
 
 CMD ["scheduler"]

airflow-scheduler.Dockerfile

@@ -30,3 +30,11 @@ AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK='true'
 _PIP_ADDITIONAL_REQUIREMENTS=${_PIP_ADDITIONAL_REQUIREMENTS:-}
 AIRFLOW_CONN_QFDMO-DJANGO-DB='postgres://qfdmo:qfdmo@lvao-db:5432/qfdmo' # pragma: allowlist secret
 DATABASE_URL=postgis://qfdmo:qfdmo@lvao-db:5432/qfdmo # pragma: allowlist secret
+
+# DBT env vars
+POSTGRES_HOST=lvao-db
+POSTGRES_PORT=5432
+POSTGRES_USER=qfdmo
+POSTGRES_PASSWORD=qfdmo
+POSTGRES_DB=qfdmo
+POSTGRES_SCHEMA=public

dags/.env.template

@@ -0,0 +1,32 @@
+from datetime import datetime, timedelta
+
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+
+default_args = {
+    "owner": "airflow",
+    "depends_on_past": False,
+    "start_date": datetime(2025, 2, 14),
+    "email_on_failure": False,
+    "email_on_retry": False,
+    "retries": 1,
+    "retry_delay": timedelta(minutes=5),
+}
+
+with DAG("dbt_dag", default_args=default_args) as dag:
+
+    run_dbt_model = BashOperator(
+        task_id="run_dbt_model",
+        bash_command=(
+            "cd /opt/airflow/dbt/ && dbt run --select qfdmo.exhaustive_acteur_views"
+        ),
+        dag=dag,
+    )
+    test_dbt_model = BashOperator(
+        task_id="test_dbt_model",
+        bash_command=(
+            "cd /opt/airflow/dbt/ && dbt test --select qfdmo.exhaustive_acteur_views"
+        ),
+        dag=dag,
+    )
+    run_dbt_model >> test_dbt_model

dags/acteur_views/dags/build_vue.py

@@ -0,0 +1,5 @@
+
+target/
+dbt_packages/
+logs/
+.user.yml

dbt/.gitignore

@@ -0,0 +1,38 @@
+# Preuve de concept d'utilisation de dbt pour la gestion des données
+
+## Installation
+
+A la racine du projet
+
+```sh
+pip install -r airflow-requirements.txt -r requirements.txt -r dev-requirements.txt
+```
+
+Puis dans le dossier dbt
+
+```sh
+cd dbt
+dbt deps
+```
+
+## Utilisation
+
+Lancer dbt dans le dossier dbt.
+L'option select permet de lancer un seul ensemble de models, cf [project.yml](./dbt_project.yml).
+
+```sh
+dbt run --select qfdmo.exhaustive_acteur_views
+```
+
+Lancer les tests
+
+```sh
+dbt run --select qfdmo.exhaustive_acteur_views
+```
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

dbt/README.md

@@ -0,0 +1,31 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'qfdmo'
+version: '1.0.0'
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'dbt_test'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  qfdmo:
+    exhaustive_acteur_views:
+      materialized: view
+
+    opendata_acteur_views:
+      materialized: view