Skip to content

Analyst/vinca nitrates #143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 33 commits into from
Apr 22, 2025
Merged

Analyst/vinca nitrates #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
0134727
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Mar 7, 2025
60970e5
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Mar 8, 2025
1fade97
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Mar 11, 2025
eeba59c
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Mar 17, 2025
df29401
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Mar 23, 2025
3489462
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Apr 3, 2025
8802ddd
nitrites : last prel per udi
20k-P Apr 3, 2025
9f79b1f
fix: valtraduite_no2 was missing
20k-P Apr 4, 2025
75c93cc
feat: add test for categorie nitrite
20k-P Apr 5, 2025
ad7e42a
feat: add last_prel nitrite into final table
20k-P Apr 5, 2025
ffa66d6
fix: update CASE WHEN for resultat
20k-P Apr 6, 2025
cdc3dee
feat: update possible values into resultat
20k-P Apr 8, 2025
b312509
feat: use new file GF + Update filed resultats
20k-P Apr 11, 2025
0fa120b
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Apr 11, 2025
63f81e6
fix: valtraduite_no3_no2_cacl into split_nitrites_with_ref
20k-P Apr 12, 2025
f30fd22
feat: check date between test to define resultat
20k-P Apr 12, 2025
97da477
Merge branch 'main' of https://github.com/dataforgoodfr/13_pollution_eau
20k-P Apr 12, 2025
7b2182f
Merge branch 'main' into analyst/vinca_nitrates
20k-P Apr 12, 2025
33a9e64
fix: pre-commit
20k-P Apr 12, 2025
2e2ef7b
fix: update test
20k-P Apr 13, 2025
7af80c1
feat: update method for nitrates/nitrites + add missing field into yaml
20k-P Apr 15, 2025
ea55eb7
Merge branch 'main' into analyst/vinca_nitrates
20k-P Apr 17, 2025
bf56307
fix
20k-P Apr 17, 2025
550edba
fix
20k-P Apr 17, 2025
cfdb0c8
fix
20k-P Apr 17, 2025
a596893
update line width configuration
jereze Apr 21, 2025
c82cf89
core: Allége le SQL
20k-P Apr 21, 2025
a27c3a9
small model updates
jereze Apr 21, 2025
53d4caf
rename
jereze Apr 21, 2025
21f3cdb
add a test
jereze Apr 21, 2025
e90bddf
Merge branch 'main' into analyst/vinca_nitrates
jereze Apr 21, 2025
0422ff8
misc
jereze Apr 21, 2025
fc3b492
misc
jereze Apr 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
360 changes: 360 additions & 0 deletions analytics/notebooks/Test_depassements_no2vsno3.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,360 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 5,
"id": "443a3c28-b272-4788-9995-0bcadc959d15",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The sql extension is already loaded. To reload it, use:\n",
" %reload_ext sql\n"
]
}
],
"source": [
"%load_ext sql\n",
"%sql duckdb:///../../database/data.duckdb\n",
"%config SqlMagic.named_parameters=\"enabled\" "
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "33985e98-d332-40dc-9038-4bd998e5567d",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"<span style=\"None\">Running query in &#x27;duckdb:///../../database/data.duckdb&#x27;</span>"
],
"text/plain": [
"Running query in 'duckdb:///../../database/data.duckdb'"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6b6b6d2428cc474fadf32706ae5d5f72",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<table>\n",
" <thead>\n",
" <tr>\n",
" <th>annee</th>\n",
" <th>nb_prelevements</th>\n",
" <th>nb_dep_3critères</th>\n",
" <th>percent_3critères</th>\n",
" <th>nb_prelevements_n02</th>\n",
" <th>nb_dep_n02</th>\n",
" <th>percent_n02</th>\n",
" <th>nb_prelevements_n03</th>\n",
" <th>nb_dep_n03</th>\n",
" <th>percent_n03</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <td>2020</td>\n",
" <td>129386</td>\n",
" <td>1745</td>\n",
" <td>1.0</td>\n",
" <td>129385</td>\n",
" <td>27</td>\n",
" <td>0.0</td>\n",
" <td>129381</td>\n",
" <td>1703</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2025</td>\n",
" <td>18138</td>\n",
" <td>285</td>\n",
" <td>1.0</td>\n",
" <td>18138</td>\n",
" <td>1</td>\n",
" <td>0.0</td>\n",
" <td>17681</td>\n",
" <td>283</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2021</td>\n",
" <td>126493</td>\n",
" <td>1823</td>\n",
" <td>1.0</td>\n",
" <td>126487</td>\n",
" <td>17</td>\n",
" <td>0.0</td>\n",
" <td>126490</td>\n",
" <td>1802</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2022</td>\n",
" <td>128745</td>\n",
" <td>1605</td>\n",
" <td>1.0</td>\n",
" <td>128736</td>\n",
" <td>16</td>\n",
" <td>0.0</td>\n",
" <td>128737</td>\n",
" <td>1586</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2024</td>\n",
" <td>127734</td>\n",
" <td>1980</td>\n",
" <td>1.0</td>\n",
" <td>127733</td>\n",
" <td>16</td>\n",
" <td>0.0</td>\n",
" <td>127733</td>\n",
" <td>1960</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2023</td>\n",
" <td>129172</td>\n",
" <td>1447</td>\n",
" <td>1.0</td>\n",
" <td>129171</td>\n",
" <td>35</td>\n",
" <td>0.0</td>\n",
" <td>129154</td>\n",
" <td>1405</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"+-------+-----------------+------------------+-------------------+---------------------+------------+-------------+---------------------+------------+-------------+\n",
"| annee | nb_prelevements | nb_dep_3critères | percent_3critères | nb_prelevements_n02 | nb_dep_n02 | percent_n02 | nb_prelevements_n03 | nb_dep_n03 | percent_n03 |\n",
"+-------+-----------------+------------------+-------------------+---------------------+------------+-------------+---------------------+------------+-------------+\n",
"| 2020 | 129386 | 1745 | 1.0 | 129385 | 27 | 0.0 | 129381 | 1703 | 1.0 |\n",
"| 2025 | 18138 | 285 | 1.0 | 18138 | 1 | 0.0 | 17681 | 283 | 1.0 |\n",
"| 2021 | 126493 | 1823 | 1.0 | 126487 | 17 | 0.0 | 126490 | 1802 | 1.0 |\n",
"| 2022 | 128745 | 1605 | 1.0 | 128736 | 16 | 0.0 | 128737 | 1586 | 1.0 |\n",
"| 2024 | 127734 | 1980 | 1.0 | 127733 | 16 | 0.0 | 127733 | 1960 | 1.0 |\n",
"| 2023 | 129172 | 1447 | 1.0 | 129171 | 35 | 0.0 | 129154 | 1405 | 1.0 |\n",
"+-------+-----------------+------------------+-------------------+---------------------+------------+-------------+---------------------+------------+-------------+"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
"WITH\n",
"prels AS (\n",
" SELECT\n",
" de_partition AS annee,\n",
" cdreseau,\n",
" referenceprel,\n",
" datetimeprel,\n",
" COUNT(DISTINCT cdparametresiseeaux) AS nb_parametres,\n",
" MAX(\n",
" CASE\n",
" WHEN\n",
" cdparametresiseeaux = 'NO3'\n",
" THEN valtraduite\n",
" ELSE 0\n",
" END\n",
" ) AS valtraduite_no3,\n",
" MAX(\n",
" CASE\n",
" WHEN\n",
" cdparametresiseeaux = 'NO2'\n",
" THEN valtraduite\n",
" ELSE 0\n",
" END\n",
" ) AS valtraduite_no2,\n",
" MAX(\n",
" CASE\n",
" WHEN\n",
" cdparametresiseeaux = 'NO3_NO2'\n",
" THEN valtraduite\n",
" ELSE 0\n",
" END\n",
" ) AS valtraduite_no3_no2\n",
" FROM\n",
" int__resultats_udi_communes\n",
" WHERE\n",
" categorie = 'nitrate'\n",
" GROUP BY\n",
" annee,\n",
" cdreseau,\n",
" referenceprel,\n",
" datetimeprel,\n",
"),\n",
"\n",
"valeur_ref AS (\n",
" SELECT\n",
" MAX(\n",
" CASE\n",
" WHEN\n",
" cdparametresiseeaux = 'NO3'\n",
" THEN limite_qualite\n",
" END\n",
" ) AS limite_qualite_no3,\n",
" MAX(\n",
" CASE\n",
" WHEN\n",
" cdparametresiseeaux = 'NO3_NO2'\n",
" THEN limite_qualite\n",
" END\n",
" ) AS limite_qualite_no3_no2,\n",
" MAX(\n",
" CASE\n",
" WHEN\n",
" cdparametresiseeaux = 'NO2'\n",
" THEN limite_qualite\n",
" END\n",
" ) AS limite_qualite_no2\n",
" FROM\n",
" int__valeurs_de_reference\n",
" WHERE\n",
" categorie_1 = 'nitrate'\n",
")\n",
"\n",
"SELECT\n",
" -- prels.cdreseau,\n",
" prels.annee,\n",
" -- 'nitrate' AS categorie,\n",
" -- 'bilan_annuel_' || prels.annee AS periode,\n",
" COUNT(DISTINCT prels.referenceprel) AS nb_prelevements,\n",
" COUNT(\n",
" DISTINCT\n",
" CASE\n",
" WHEN\n",
" prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n",
" OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n",
" OR prels.valtraduite_no3_no2\n",
" >= valeur_ref.limite_qualite_no3_no2\n",
" OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3\n",
" >= valeur_ref.limite_qualite_no3_no2\n",
" THEN prels.referenceprel\n",
" END\n",
" ) AS nb_dep_3critères,\n",
" trunc((\n",
" COUNT(\n",
" DISTINCT\n",
" CASE\n",
" WHEN\n",
" prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n",
" OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n",
" OR prels.valtraduite_no3_no2\n",
" >= valeur_ref.limite_qualite_no3_no2\n",
" OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3\n",
" >= valeur_ref.limite_qualite_no3_no2\n",
" THEN prels.referenceprel\n",
" END\n",
" )::float\n",
" /\n",
" COUNT(DISTINCT prels.referenceprel)::float\n",
" )*100) AS percent_3critères,\n",
"\n",
" COUNT(DISTINCT CASE WHEN valtraduite_no2 IS NOT NULL THEN prels.referenceprel END) AS nb_prelevements_n02,\n",
" COUNT(\n",
" DISTINCT\n",
" CASE\n",
" WHEN\n",
" prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n",
" THEN prels.referenceprel\n",
" END\n",
" ) AS nb_dep_n02,\n",
" trunc((\n",
" COUNT(\n",
" DISTINCT\n",
" CASE\n",
" WHEN\n",
" prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n",
" THEN prels.referenceprel\n",
" END\n",
" )::float\n",
" /\n",
" COUNT(DISTINCT CASE WHEN valtraduite_no2 IS NOT NULL THEN prels.referenceprel END)::float\n",
" )*100) AS percent_n02,\n",
"\n",
" COUNT(DISTINCT CASE WHEN valtraduite_no3 IS NOT NULL THEN prels.referenceprel END) AS nb_prelevements_n03,\n",
" COUNT(\n",
" DISTINCT\n",
" CASE\n",
" WHEN\n",
" prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n",
" THEN prels.referenceprel\n",
" END\n",
" ) AS nb_dep_n03,\n",
" trunc((\n",
" COUNT(\n",
" DISTINCT\n",
" CASE\n",
" WHEN\n",
" prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n",
" THEN prels.referenceprel\n",
" END\n",
" )::float\n",
" /\n",
" COUNT(DISTINCT CASE WHEN valtraduite_no2 IS NOT NULL THEN prels.referenceprel END)::float\n",
" )*100) AS percent_n03\n",
"\n",
"FROM prels\n",
"CROSS JOIN valeur_ref\n",
"GROUP BY /*prels.cdreseau,*/ prels.annee\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d2b49cfb-e586-4523-991e-41a25a7d7b02",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading