Skip to content

Commit 4b8304a

Browse files
kolokkolok
committed
reorganisation par layer
1 parent 9c84f0d commit 4b8304a

File tree

87 files changed

+1791
-477
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+1791
-477
lines changed

‎dags/acteur_views/dags/build_vue.py‎

Lines changed: 153 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -25,51 +25,167 @@
2525
schedule=None,
2626
max_active_runs=1,
2727
) as dag:
28+
"""
29+
Run DBT models
30+
dbt run --models base
31+
dbt test --models base
2832
29-
run_dbt_exhaustive_acteurs_model = BashOperator(
30-
task_id="build_exhaustive_acteurs",
31-
bash_command=(
32-
"cd /opt/airflow/dbt/ && dbt run --select qfdmo.exhaustive_acteurs"
33-
),
33+
dbt run --models intermediate
34+
dbt test --models intermediate
35+
36+
dbt run --models marts.exhaustive
37+
dbt test --models marts.exhaustive
38+
dbt run --models marts.carte
39+
dbt test --models marts.carte
40+
dbt run --models marts.opendata
41+
dbt test --models marts.opendata
42+
43+
dbt run --models exposure.exhaustive
44+
dbt test --models exposure.exhaustive
45+
dbt run --models exposure.carte
46+
dbt test --models exposure.carte
47+
dbt run --models exposure.opendata
48+
dbt test --models exposure.opendata
49+
50+
"""
51+
dbt_run_base = BashOperator(
52+
task_id="run_base",
53+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models base"),
3454
dag=dag,
3555
)
36-
test_dbt_exhaustive_acteurs_model = BashOperator(
37-
task_id="test_exhaustive_acteurs",
38-
bash_command=(
39-
"cd /opt/airflow/dbt/ && dbt test --select qfdmo.exhaustive_acteurs"
40-
),
41-
dag=dag,
56+
dbt_test_base = BashOperator(
57+
task_id="test_base",
58+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models base"),
59+
# dag=dag,
4260
)
43-
run_dbt_carte_acteurs_model = BashOperator(
44-
task_id="build_carte_acteurs",
45-
bash_command=("cd /opt/airflow/dbt/ && dbt run --select qfdmo.carte_acteurs"),
46-
dag=dag,
61+
62+
dbt_run_intermediate = BashOperator(
63+
task_id="run_intermediate",
64+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models intermediate"),
65+
# dag=dag,
4766
)
48-
test_dbt_carte_acteurs_model = BashOperator(
49-
task_id="test_carte_acteurs",
50-
bash_command=("cd /opt/airflow/dbt/ && dbt test --select qfdmo.carte_acteurs"),
51-
dag=dag,
67+
dbt_test_intermediate = BashOperator(
68+
task_id="test_intermediate",
69+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models intermediate"),
70+
# dag=dag,
5271
)
53-
run_dbt_opendata_acteurs_model = BashOperator(
54-
task_id="build_opendata_acteurs",
55-
bash_command=(
56-
"cd /opt/airflow/dbt/ && dbt run --select qfdmo.opendata_acteurs"
57-
),
58-
dag=dag,
72+
73+
dbt_run_marts_exhaustive = BashOperator(
74+
task_id="run_marts_exhaustive",
75+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models marts.exhaustive"),
76+
# dag=dag,
5977
)
60-
test_dbt_opendata_acteurs_model = BashOperator(
61-
task_id="test_opendata_acteurs",
62-
bash_command=(
63-
"cd /opt/airflow/dbt/ && dbt test --select qfdmo.opendata_acteurs"
64-
),
65-
dag=dag,
78+
dbt_test_marts_exhaustive = BashOperator(
79+
task_id="test_marts_exhaustive",
80+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models marts.exhaustive"),
81+
# dag=dag,
82+
)
83+
dbt_run_exposure_exhaustive = BashOperator(
84+
task_id="run_exposure_exhaustive",
85+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models exposure.exhaustive"),
86+
# dag=dag,
87+
)
88+
dbt_test_exposure_exhaustive = BashOperator(
89+
task_id="test_exposure_exhaustive",
90+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models exposure.exhaustive"),
91+
# dag=dag,
92+
)
93+
94+
dbt_run_marts_carte = BashOperator(
95+
task_id="run_marts_carte",
96+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models marts.carte"),
97+
# dag=dag,
98+
)
99+
dbt_test_marts_carte = BashOperator(
100+
task_id="test_marts_carte",
101+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models marts.carte"),
102+
# dag=dag,
103+
)
104+
dbt_run_exposure_carte = BashOperator(
105+
task_id="run_exposure_carte",
106+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models exposure.carte"),
107+
# dag=dag,
108+
)
109+
dbt_test_exposure_carte = BashOperator(
110+
task_id="test_exposure_carte",
111+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models exposure.carte"),
112+
# dag=dag,
113+
)
114+
115+
dbt_run_marts_opendata = BashOperator(
116+
task_id="run_marts_opendata",
117+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models marts.opendata"),
118+
# dag=dag,
119+
)
120+
dbt_test_marts_opendata = BashOperator(
121+
task_id="test_marts_opendata",
122+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models marts.opendata"),
123+
# dag=dag,
124+
)
125+
dbt_run_exposure_opendata = BashOperator(
126+
task_id="run_exposure_opendata",
127+
bash_command=("cd /opt/airflow/dbt/ && dbt run --models exposure.opendata"),
128+
# dag=dag,
129+
)
130+
dbt_test_exposure_opendata = BashOperator(
131+
task_id="test_exposure_opendata",
132+
bash_command=("cd /opt/airflow/dbt/ && dbt test --models exposure.opendata"),
133+
# dag=dag,
134+
)
135+
136+
# Définir la séquence principale
137+
dbt_run_base >> dbt_test_base >> dbt_run_intermediate >> dbt_test_intermediate
138+
139+
# Après intermediate, brancher en parallèle
140+
# Branche exhaustive
141+
(
142+
dbt_test_intermediate
143+
>> dbt_run_marts_exhaustive
144+
>> dbt_test_marts_exhaustive
145+
>> dbt_run_exposure_exhaustive
146+
>> dbt_test_exposure_exhaustive
147+
)
148+
149+
# Branche carte
150+
(
151+
dbt_test_intermediate
152+
>> dbt_run_marts_carte
153+
>> dbt_test_marts_carte
154+
>> dbt_run_exposure_carte
155+
>> dbt_test_exposure_carte
66156
)
67157

158+
# Branche opendata
68159
(
69-
run_dbt_exhaustive_acteurs_model
70-
>> test_dbt_exhaustive_acteurs_model
71-
>> run_dbt_carte_acteurs_model
72-
>> test_dbt_carte_acteurs_model
73-
>> run_dbt_opendata_acteurs_model
74-
>> test_dbt_opendata_acteurs_model
160+
dbt_test_intermediate
161+
>> dbt_run_marts_opendata
162+
>> dbt_test_marts_opendata
163+
>> dbt_run_exposure_opendata
164+
>> dbt_test_exposure_opendata
75165
)
166+
# chain(
167+
# dbt_run_base,
168+
# dbt_test_base,
169+
# dbt_run_intermediate,
170+
# dbt_test_intermediate,
171+
# [
172+
# chain(
173+
# dbt_run_marts_exhaustive,
174+
# dbt_test_marts_exhaustive,
175+
# dbt_run_exposure_exhaustive,
176+
# dbt_test_exposure_exhaustive,
177+
# ),
178+
# chain(
179+
# dbt_run_marts_carte,
180+
# dbt_test_marts_carte,
181+
# dbt_run_exposure_carte,
182+
# dbt_test_exposure_carte,
183+
# ),
184+
# chain(
185+
# dbt_run_marts_opendata,
186+
# dbt_test_marts_opendata,
187+
# dbt_run_exposure_opendata,
188+
# dbt_test_exposure_opendata,
189+
# ),
190+
# ],
191+
# )

‎dbt/dbt_project.yml‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,25 @@ clean-targets:
2323
- "dbt_packages"
2424

2525
models:
26-
qfdmo:
27-
exhaustive_acteurs:
28-
materialized: table
29-
30-
opendata_acteurs:
31-
materialized: table
32-
33-
carte_acteurs:
34-
materialized: table
26+
base:
27+
schema: public
28+
+materialized: view
29+
intermediate:
30+
schema: public
31+
+materialized: view
32+
marts:
33+
+materialized: table
34+
exhaustive:
35+
schema: public
36+
carte:
37+
schema: public
38+
opendata:
39+
schema: public
40+
exposure:
41+
+materialized: table
42+
exhaustive:
43+
schema: public
44+
carte:
45+
schema: public
46+
opendata:
47+
schema: public

‎dbt/macros/macro_acteur.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{%- macro acteur(ephemeral_filtered_acteur, propositionservice ) -%}
2+
3+
SELECT DISTINCT va.*
4+
FROM {{ ref(ephemeral_filtered_acteur) }} AS va
5+
INNER JOIN {{ ref(propositionservice) }} AS cps
6+
ON va.identifiant_unique = cps.acteur_id
7+
8+
{%- endmacro -%}

‎dbt/models/opendata_acteurs/opendata_acteur_acteur_services.sql‎ renamed to ‎dbt/macros/macro_acteur_acteur_services.sql‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
1+
{%- macro acteur_acteur_services(ephemeral_filtered_acteur, acteur ) -%}
2+
13
WITH nochild_acteur_acteur_services AS (
24
SELECT
35
aas.vueacteur_id AS acteur_id,
46
aas.acteurservice_id AS acteurservice_id
57
FROM qfdmo_vueacteur_acteur_services aas
6-
INNER JOIN {{ ref('temp_opendata_filteredacteur') }} AS a ON aas.vueacteur_id = a.identifiant_unique AND a.parent_id is null
8+
INNER JOIN {{ ref(ephemeral_filtered_acteur) }} AS a ON aas.vueacteur_id = a.identifiant_unique AND a.parent_id is null
79
GROUP BY aas.vueacteur_id, aas.acteurservice_id
810
),
911
parentacteur_acteur_services AS (
1012
SELECT
1113
a.parent_id AS acteur_id,
1214
aas.acteurservice_id AS acteurservice_id
1315
FROM qfdmo_vueacteur_acteur_services aas
14-
INNER JOIN {{ ref('temp_opendata_filteredacteur') }} AS a ON aas.vueacteur_id = a.identifiant_unique AND a.parent_id is not null
16+
INNER JOIN {{ ref(ephemeral_filtered_acteur) }} AS a ON aas.vueacteur_id = a.identifiant_unique AND a.parent_id is not null
1517
GROUP BY a.parent_id, aas.acteurservice_id
1618
),
1719
acteur_acteur_services AS (
@@ -22,4 +24,6 @@ acteur_acteur_services AS (
2224

2325
SELECT ROW_NUMBER() OVER (ORDER BY acteur_id, aas.acteurservice_id) AS id, aas.*
2426
FROM acteur_acteur_services AS aas
25-
INNER JOIN {{ ref('opendata_acteur') }} AS a ON a.identifiant_unique = acteur_id
27+
INNER JOIN {{ ref(acteur) }} AS a ON a.identifiant_unique = acteur_id
28+
29+
{%- endmacro -%}

‎dbt/models/opendata_acteurs/opendata_acteur_labels.sql‎ renamed to ‎dbt/macros/macro_acteur_labels.sql‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
1+
{%- macro acteur_labels(ephemeral_filtered_acteur, acteur ) -%}
2+
13
WITH nochild_acteur_labels AS (
24
SELECT
35
al.vueacteur_id AS acteur_id,
46
al.labelqualite_id AS labelqualite_id
57
FROM qfdmo_vueacteur_labels al
6-
INNER JOIN {{ ref('temp_opendata_filteredacteur') }} AS a ON al.vueacteur_id = a.identifiant_unique AND a.parent_id is null
8+
INNER JOIN {{ ref(ephemeral_filtered_acteur) }} AS a ON al.vueacteur_id = a.identifiant_unique AND a.parent_id is null
79
GROUP BY al.vueacteur_id, al.labelqualite_id
810
),
911
parentacteur_labels AS (
1012
SELECT
1113
a.parent_id AS acteur_id,
1214
al.labelqualite_id AS labelqualite_id
1315
FROM qfdmo_vueacteur_labels al
14-
INNER JOIN {{ ref('temp_opendata_filteredacteur') }} AS a ON al.vueacteur_id = a.identifiant_unique AND a.parent_id is not null
16+
INNER JOIN {{ ref(ephemeral_filtered_acteur) }} AS a ON al.vueacteur_id = a.identifiant_unique AND a.parent_id is not null
1517
GROUP BY a.parent_id, al.labelqualite_id
1618
),
1719
acteur_labels AS (
@@ -22,4 +24,6 @@ acteur_labels AS (
2224

2325
SELECT ROW_NUMBER() OVER (ORDER BY acteur_id, al.labelqualite_id) AS id, al.*
2426
FROM acteur_labels AS al
25-
INNER JOIN {{ ref('opendata_acteur') }} AS a ON a.identifiant_unique = acteur_id
27+
INNER JOIN {{ ref(acteur) }} AS a ON a.identifiant_unique = acteur_id
28+
29+
{%- endmacro -%}

‎dbt/models/opendata_acteurs/opendata_acteur_sources.sql‎ renamed to ‎dbt/macros/macro_acteur_sources.sql‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
1+
{%- macro acteur_sources(ephemeral_filtered_acteur, acteur ) -%}
2+
13
WITH nochild_acteur_labels AS (
24
SELECT
35
a.identifiant_unique AS acteur_id,
46
a.source_id AS source_id
5-
FROM {{ ref('temp_opendata_filteredacteur') }} AS a
7+
FROM {{ ref(ephemeral_filtered_acteur) }} AS a
68
WHERE a.parent_id is null AND a.source_id is not null
79
GROUP BY a.identifiant_unique, a.source_id
810
),
911
parentacteur_labels AS (
1012
SELECT
1113
a.parent_id AS acteur_id,
1214
a.source_id AS source_id
13-
FROM {{ ref('temp_opendata_filteredacteur') }} AS a
15+
FROM {{ ref(ephemeral_filtered_acteur) }} AS a
1416
WHERE a.parent_id is not null
1517
GROUP BY a.parent_id, a.source_id
1618
),
@@ -22,4 +24,6 @@ acteur_sources AS (
2224

2325
SELECT ROW_NUMBER() OVER (ORDER BY acteur_id, s.source_id) AS id, s.*
2426
FROM acteur_sources AS s
25-
INNER JOIN {{ ref('opendata_acteur') }} AS a ON a.identifiant_unique = acteur_id
27+
INNER JOIN {{ ref(acteur) }} AS a ON a.identifiant_unique = acteur_id
28+
29+
{%- endmacro -%}

‎dbt/models/opendata_acteurs/temp_opendata_parentpropositionservice.sql‎ renamed to ‎dbt/macros/macro_filtered_parentpropositionservice.sql‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1+
{%- macro filtered_parentpropositionservice(ephemeral_filtered_acteur ) -%}
12

23
SELECT
34
tvps.id AS id,
45
tcfa.parent_id AS parent_id,
56
tvps.acteur_id AS acteur_id,
67
tvps.action_id AS action_id
78
FROM qfdmo_vuepropositionservice AS tvps
8-
INNER JOIN {{ ref('temp_opendata_filteredacteur') }} AS tcfa
9+
INNER JOIN {{ ref(ephemeral_filtered_acteur) }} AS tcfa
910
ON tvps.acteur_id = tcfa.identifiant_unique
1011
AND tcfa.parent_id IS NOT NULL
12+
13+
{%- endmacro -%}

‎dbt/models/carte_acteurs/temp_propositionservice.sql‎ renamed to ‎dbt/macros/macro_filtered_propositionservice.sql‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
{%- macro filtered_propositionservice(ephemeral_filtered_acteur, ephemeral_filtered_parentpropositionservice ) -%}
2+
13
with parent_propositionservice AS (
24
SELECT
35
concat(pps.parent_id::text, '_', pps.action_id::text) AS id,
46
pps.parent_id AS acteur_id,
57
pps.action_id AS action_id
6-
FROM {{ ref('temp_parentpropositionservice') }} AS pps
8+
FROM {{ ref(ephemeral_filtered_parentpropositionservice) }} AS pps
79
group by 2,3
810
),
911
nochild_propositionservice AS (
@@ -12,10 +14,12 @@ nochild_propositionservice AS (
1214
vps.acteur_id AS acteur_id,
1315
vps.action_id AS action_id
1416
FROM qfdmo_vuepropositionservice AS vps
15-
INNER JOIN {{ ref('temp_filteredacteur') }} AS cfa
17+
INNER JOIN {{ ref(ephemeral_filtered_acteur) }} AS cfa
1618
ON vps.acteur_id = cfa.identifiant_unique AND cfa.parent_id is null
1719
)
1820

1921
SELECT * FROM parent_propositionservice
2022
union all
2123
SELECT * FROM nochild_propositionservice
24+
25+
{%- endmacro -%}

‎dbt/macros/macro_propositionservice.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{%- macro propositionservice(ephemeral_filtered_propositionservice, propositionservice_sous_categories ) -%}
2+
3+
SELECT
4+
MIN(ps.id) AS id,
5+
ps.acteur_id,
6+
ps.action_id
7+
FROM {{ ref(ephemeral_filtered_propositionservice) }} AS ps
8+
INNER JOIN {{ ref(propositionservice_sous_categories) }} AS pssscat
9+
ON ps.id = pssscat.propositionservice_id
10+
GROUP BY acteur_id, action_id
11+
12+
{%- endmacro -%}

0 commit comments

Comments
 (0)