From 8802ddded1c08811f0098e357807e734c2c9948a Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Thu, 3 Apr 2025 13:21:22 +0200 Subject: [PATCH 01/30] nitrites : last prel per udi --- analytics/notebooks/resultats_nitrates.ipynb | 601 ++++++++++++++++++ .../nitrites/_int__nitrites_models.yaml | 40 ++ .../int__resultats_nitrites_udi_dernier.sql | 77 +++ 3 files changed, 718 insertions(+) create mode 100644 analytics/notebooks/resultats_nitrates.ipynb create mode 100644 dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml create mode 100644 dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb new file mode 100644 index 00000000..71dcf1ab --- /dev/null +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -0,0 +1,601 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2e8d5bd2-d7c1-4ecb-805d-27eaa75195e0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "The 'toml' package isn't installed. To load settings from pyproject.toml or ~/.jupysql/config, install with: pip install toml" + ], + "text/plain": [ + "The 'toml' package isn't installed. To load settings from pyproject.toml or ~/.jupysql/config, install with: pip install toml" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Connecting to 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Connecting to 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%load_ext sql\n", + "%sql duckdb:///../../database/data.duckdb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3e1967e8-eba0-4261-bf9c-d16dd4a9b271", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdparametresiseeauxcdparametrelibmajparametrelibminparametrecasparamcategorie
NO31340NITRATES (EN NO3)Nitrates (en NO3)Nonenitrite
NO3_NO26374NITRATES/50 + NITRITES/3Nitrates/50 + Nitrites/3Nonenitrite
NO21339NITRITES (EN NO2)Nitrites (en NO2)Nonenitrite
NO31340NITRATES (EN NO3)Nitrates (en NO3)14797-55-8nitrite
NO21339NITRITES (EN NO2)Nitrites (en NO2)14797-65-0nitrite
" + ], + "text/plain": [ + "+---------------------+-------------+--------------------------+--------------------------+------------+-----------+\n", + "| cdparametresiseeaux | cdparametre | libmajparametre | libminparametre | casparam | categorie |\n", + "+---------------------+-------------+--------------------------+--------------------------+------------+-----------+\n", + "| NO3 | 1340 | NITRATES (EN NO3) | Nitrates (en NO3) | None | nitrite |\n", + "| NO3_NO2 | 6374 | NITRATES/50 + NITRITES/3 | Nitrates/50 + Nitrites/3 | None | nitrite |\n", + "| NO2 | 1339 | NITRITES (EN NO2) | Nitrites (en NO2) | None | nitrite |\n", + "| NO3 | 1340 | NITRATES (EN NO3) | Nitrates (en NO3) | 14797-55-8 | nitrite |\n", + "| NO2 | 1339 | NITRITES (EN NO2) | Nitrites (en NO2) | 14797-65-0 | nitrite |\n", + "+---------------------+-------------+--------------------------+--------------------------+------------+-----------+" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "SELECT * FROM mapping_categories WHERE categorie='nitrite'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "de799aa8-64a7-4423-901c-7de0fc237b70", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6cbe026e692442e0af1a01930f5c86cc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000260nitritedernier_prel2024-12-06 12:02:003non_conforme
001000268nitritedernier_prel2024-11-28 08:25:003non_conforme
001000275nitritedernier_prel2024-07-17 09:30:003non_quantifie
001000293nitritedernier_prel2024-10-10 10:20:003non_conforme
001000374nitritedernier_prel2024-05-22 09:40:003non_conforme
001000539nitritedernier_prel2024-12-09 15:22:003non_conforme
001000541nitritedernier_prel2024-12-03 11:55:003non_conforme
001000578nitritedernier_prel2024-11-29 08:45:003non_conforme
001000792nitritedernier_prel2024-11-21 10:12:003non_conforme
001000846nitritedernier_prel2024-11-22 10:53:003non_conforme
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "| 001000260 | nitrite | dernier_prel | 2024-12-06 12:02:00 | 3 | non_conforme |\n", + "| 001000268 | nitrite | dernier_prel | 2024-11-28 08:25:00 | 3 | non_conforme |\n", + "| 001000275 | nitrite | dernier_prel | 2024-07-17 09:30:00 | 3 | non_quantifie |\n", + "| 001000293 | nitrite | dernier_prel | 2024-10-10 10:20:00 | 3 | non_conforme |\n", + "| 001000374 | nitrite | dernier_prel | 2024-05-22 09:40:00 | 3 | non_conforme |\n", + "| 001000539 | nitrite | dernier_prel | 2024-12-09 15:22:00 | 3 | non_conforme |\n", + "| 001000541 | nitrite | dernier_prel | 2024-12-03 11:55:00 | 3 | non_conforme |\n", + "| 001000578 | nitrite | dernier_prel | 2024-11-29 08:45:00 | 3 | non_conforme |\n", + "| 001000792 | nitrite | dernier_prel | 2024-11-21 10:12:00 | 3 | non_conforme |\n", + "| 001000846 | nitrite | dernier_prel | 2024-11-22 10:53:00 | 3 | non_conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --save int__resultats_nitrites_udi_dernier\n", + "WITH\n", + "last_pvl AS (\n", + " SELECT\n", + " cdreseau,\n", + " categorie,\n", + " cdparametresiseeaux,\n", + " datetimeprel,\n", + " valtraduite,\n", + " ROW_NUMBER()\n", + " OVER (\n", + " PARTITION BY cdreseau, cdparametresiseeaux\n", + " ORDER BY datetimeprel DESC\n", + " )\n", + " AS row_number\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrite'\n", + " AND\n", + " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + "),\n", + "\n", + "split_nitrites AS (\n", + " SELECT\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie,\n", + " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", + " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", + " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3' THEN last_pvl.valtraduite END) AS valtraduite_no3,\n", + " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite END) AS valtraduite_no3_no2,\n", + " FROM\n", + " last_pvl\n", + " WHERE\n", + " last_pvl.row_number = 1\n", + " GROUP BY \n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie \n", + ")\n", + "\n", + "SELECT\n", + " split_nitrites.cdreseau,\n", + " split_nitrites.categorie,\n", + " 'dernier_prel' AS periode,\n", + " split_nitrites.dernier_prel_datetime,\n", + " split_nitrites.nb_parametres,\n", + " CASE\n", + " WHEN\n", + " split_nitrites.valtraduite_no3 IS NULL \n", + " OR split_nitrites.valtraduite_no3_no2 IS NULL\n", + " OR split_nitrites.valtraduite_no3 = 0\n", + " OR split_nitrites.valtraduite_no3_no2 = 0\n", + " THEN 'non_quantifie'\n", + " WHEN\n", + " split_nitrites.valtraduite_no3 <0.5\n", + " AND split_nitrites.valtraduite_no3_no2 <1\n", + " THEN 'conforme'\n", + " WHEN\n", + " split_nitrites.valtraduite_no3 >= 0.5\n", + " OR split_nitrites.valtraduite_no3_no2 >= 1\n", + " THEN 'non_conforme'\n", + " ELSE 'error'\n", + " END AS resultat\n", + "FROM\n", + " split_nitrites\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2a287b57-fa15-4939-8209-7dcdf0e75eda", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4390b15f5dab4a1e9c25e33ede1ea674", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
" + ], + "text/plain": [ + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "+----------+-----------+---------+-----------------------+---------------+----------+" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier\n", + "SELECT \n", + " * \n", + "FROM \n", + " int__resultats_nitrites_udi_dernier\n", + "WHERE \n", + " resultat='error'" + ] + }, + { + "cell_type": "markdown", + "id": "c596b48d-ebc4-4fb9-aab8-b626076792d2", + "metadata": {}, + "source": [ + "# Check errors" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2137dd22-ec22-4b37-93ee-30b75c06ad42", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6b42c924850141e2b4be93ae13e43e86", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorienb_parametresdernier_prel_datetimevaltraduite_no3valtraduite_no3_no2
973000028nitrite22024-12-02 11:30:000.000None
088002296nitrite12024-09-27 12:00:00NoneNone
073002059nitrite12024-10-23 09:28:00NoneNone
" + ], + "text/plain": [ + "+-----------+-----------+---------------+-----------------------+-----------------+---------------------+\n", + "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | valtraduite_no3 | valtraduite_no3_no2 |\n", + "+-----------+-----------+---------------+-----------------------+-----------------+---------------------+\n", + "| 973000028 | nitrite | 2 | 2024-12-02 11:30:00 | 0.000 | None |\n", + "| 088002296 | nitrite | 1 | 2024-09-27 12:00:00 | None | None |\n", + "| 073002059 | nitrite | 1 | 2024-10-23 09:28:00 | None | None |\n", + "+-----------+-----------+---------------+-----------------------+-----------------+---------------------+" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --save int__resultats_nitrites_udi_dernier\n", + "WITH\n", + "last_pvl AS (\n", + " SELECT\n", + " cdreseau,\n", + " categorie,\n", + " cdparametresiseeaux,\n", + " datetimeprel,\n", + " valtraduite,\n", + " ROW_NUMBER()\n", + " OVER (\n", + " PARTITION BY cdreseau, cdparametresiseeaux\n", + " ORDER BY datetimeprel DESC\n", + " )\n", + " AS row_number\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrite'\n", + " AND\n", + " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + ")\n", + " SELECT\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie,\n", + " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", + " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", + " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3' THEN last_pvl.valtraduite END) AS valtraduite_no3,\n", + " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite END) AS valtraduite_no3_no2,\n", + " FROM\n", + " last_pvl\n", + " WHERE\n", + " last_pvl.row_number = 1\n", + " AND cdreseau IN ('088002296','073002059','973000028')\n", + " GROUP BY \n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "761be547-f21e-4712-a898-4175f8faa89c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml new file mode 100644 index 00000000..261d349b --- /dev/null +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -0,0 +1,40 @@ +version: 2 + +models: + - name: int__resultats_nitrites_udi_dernier + description: Resultats nitrites par UDI pour la temporalité 'derniers prélèvements' + columns: + - name: cdreseau + tests: + - not_null + - unique + - name: categorie + tests: + - not_null + - accepted_values: + values: ["nitrite"] + - name: periode + type: VARCHAR + tests: + - accepted_values: + values: ["dernier_prel"] + - name: dernier_prel_datetime + type: TIMESTAMP + tests: + - not_null + - dbt_utils.expression_is_true: + expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" + - name: dernier_prel_valeur + type: FLOAT + - name: nb_parametres + type: INT + tests: + - not_null + - name: resultat + tests: + - not_null + - accepted_values: + values: + - non_quantifie + - conforme + - non_conforme \ No newline at end of file diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql new file mode 100644 index 00000000..9811a967 --- /dev/null +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -0,0 +1,77 @@ +WITH +last_pvl AS ( + SELECT + cdreseau, + categorie, + cdparametresiseeaux, + datetimeprel, + valtraduite, + ROW_NUMBER() + OVER ( + PARTITION BY cdreseau, cdparametresiseeaux + ORDER BY datetimeprel DESC + ) + AS row_number + FROM + {{ ref('int__resultats_udi_communes') }} + WHERE + categorie = 'nitrite' + AND + -- On garde les prélèvements de moins d'un an + CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR +), + +split_nitrites AS ( + SELECT + last_pvl.cdreseau, + last_pvl.categorie, + COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres, + MAX(last_pvl.datetimeprel) AS dernier_prel_datetime, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3' + THEN last_pvl.valtraduite + END + ) AS valtraduite_no3, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3_NO2' + THEN last_pvl.valtraduite + END + ) AS valtraduite_no3_no2 + FROM + last_pvl + WHERE + last_pvl.row_number = 1 + GROUP BY + last_pvl.cdreseau, + last_pvl.categorie +) + +SELECT + split_nitrites.cdreseau, + split_nitrites.categorie, + 'dernier_prel' AS periode, + split_nitrites.dernier_prel_datetime, + split_nitrites.nb_parametres, + CASE + WHEN + split_nitrites.valtraduite_no3 IS NULL + OR split_nitrites.valtraduite_no3_no2 IS NULL + OR split_nitrites.valtraduite_no3 = 0 + OR split_nitrites.valtraduite_no3_no2 = 0 + THEN 'non_quantifie' + WHEN + split_nitrites.valtraduite_no3 < 0.5 + AND split_nitrites.valtraduite_no3_no2 < 1 + THEN 'conforme' + WHEN + split_nitrites.valtraduite_no3 >= 0.5 + OR split_nitrites.valtraduite_no3_no2 >= 1 + THEN 'non_conforme' + ELSE 'error' + END AS resultat +FROM + split_nitrites From 9f79b1fb92ec1a88f5e53757438ef724c972efa9 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 4 Apr 2025 10:45:18 +0200 Subject: [PATCH 02/30] fix: valtraduite_no2 was missing --- analytics/notebooks/resultats_nitrates.ipynb | 567 +++++++++++++++--- .../int__resultats_nitrites_udi_dernier.sql | 32 +- 2 files changed, 499 insertions(+), 100 deletions(-) diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb index 71dcf1ab..ee16eafe 100644 --- a/analytics/notebooks/resultats_nitrates.ipynb +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "id": "de799aa8-64a7-4423-901c-7de0fc237b70", "metadata": {}, "outputs": [ @@ -157,7 +157,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6cbe026e692442e0af1a01930f5c86cc", + "model_id": "5d6d76c002514968b669f2488a28e708", "version_major": 2, "version_minor": 0 }, @@ -184,108 +184,108 @@ " \n", " \n", " \n", - " 001000260\n", + " 001000288\n", " nitrite\n", " dernier_prel\n", - " 2024-12-06 12:02:00\n", + " 2024-11-27 11:52:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000268\n", + " 001000330\n", " nitrite\n", " dernier_prel\n", - " 2024-11-28 08:25:00\n", + " 2024-12-13 08:49:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000275\n", + " 001000337\n", " nitrite\n", " dernier_prel\n", - " 2024-07-17 09:30:00\n", + " 2024-07-04 08:31:00\n", " 3\n", - " non_quantifie\n", + " conforme\n", " \n", " \n", - " 001000293\n", + " 001000375\n", " nitrite\n", " dernier_prel\n", - " 2024-10-10 10:20:00\n", + " 2024-08-28 10:11:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000374\n", + " 001000459\n", " nitrite\n", " dernier_prel\n", - " 2024-05-22 09:40:00\n", + " 2024-07-11 10:51:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000539\n", + " 001000526\n", " nitrite\n", " dernier_prel\n", - " 2024-12-09 15:22:00\n", + " 2024-08-05 08:39:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000541\n", + " 001000592\n", " nitrite\n", " dernier_prel\n", - " 2024-12-03 11:55:00\n", + " 2024-11-28 10:22:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000578\n", + " 001000607\n", " nitrite\n", " dernier_prel\n", - " 2024-11-29 08:45:00\n", + " 2024-12-09 08:11:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000792\n", + " 001000814\n", " nitrite\n", " dernier_prel\n", - " 2024-11-21 10:12:00\n", + " 2024-12-30 11:01:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", - " 001000846\n", + " 001000823\n", " nitrite\n", " dernier_prel\n", - " 2024-11-22 10:53:00\n", + " 2024-10-18 09:24:00\n", " 3\n", - " non_conforme\n", + " conforme\n", " \n", " \n", "\n", "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", - "| 001000260 | nitrite | dernier_prel | 2024-12-06 12:02:00 | 3 | non_conforme |\n", - "| 001000268 | nitrite | dernier_prel | 2024-11-28 08:25:00 | 3 | non_conforme |\n", - "| 001000275 | nitrite | dernier_prel | 2024-07-17 09:30:00 | 3 | non_quantifie |\n", - "| 001000293 | nitrite | dernier_prel | 2024-10-10 10:20:00 | 3 | non_conforme |\n", - "| 001000374 | nitrite | dernier_prel | 2024-05-22 09:40:00 | 3 | non_conforme |\n", - "| 001000539 | nitrite | dernier_prel | 2024-12-09 15:22:00 | 3 | non_conforme |\n", - "| 001000541 | nitrite | dernier_prel | 2024-12-03 11:55:00 | 3 | non_conforme |\n", - "| 001000578 | nitrite | dernier_prel | 2024-11-29 08:45:00 | 3 | non_conforme |\n", - "| 001000792 | nitrite | dernier_prel | 2024-11-21 10:12:00 | 3 | non_conforme |\n", - "| 001000846 | nitrite | dernier_prel | 2024-11-22 10:53:00 | 3 | non_conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", + "| 001000288 | nitrite | dernier_prel | 2024-11-27 11:52:00 | 3 | conforme |\n", + "| 001000330 | nitrite | dernier_prel | 2024-12-13 08:49:00 | 3 | conforme |\n", + "| 001000337 | nitrite | dernier_prel | 2024-07-04 08:31:00 | 3 | conforme |\n", + "| 001000375 | nitrite | dernier_prel | 2024-08-28 10:11:00 | 3 | conforme |\n", + "| 001000459 | nitrite | dernier_prel | 2024-07-11 10:51:00 | 3 | conforme |\n", + "| 001000526 | nitrite | dernier_prel | 2024-08-05 08:39:00 | 3 | conforme |\n", + "| 001000592 | nitrite | dernier_prel | 2024-11-28 10:22:00 | 3 | conforme |\n", + "| 001000607 | nitrite | dernier_prel | 2024-12-09 08:11:00 | 3 | conforme |\n", + "| 001000814 | nitrite | dernier_prel | 2024-12-30 11:01:00 | 3 | conforme |\n", + "| 001000823 | nitrite | dernier_prel | 2024-10-18 09:24:00 | 3 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", "Truncated to displaylimit of 10." ] }, - "execution_count": 6, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -311,6 +311,7 @@ " WHERE\n", " categorie = 'nitrite'\n", " AND\n", + " -- On garde les prélèvements de moins d'un an\n", " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", "),\n", "\n", @@ -320,15 +321,37 @@ " last_pvl.categorie,\n", " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", - " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3' THEN last_pvl.valtraduite END) AS valtraduite_no3,\n", - " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite END) AS valtraduite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN last_pvl.valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3_no2\n", " FROM\n", " last_pvl\n", " WHERE\n", " last_pvl.row_number = 1\n", - " GROUP BY \n", + " GROUP BY\n", " last_pvl.cdreseau,\n", - " last_pvl.categorie \n", + " last_pvl.categorie\n", ")\n", "\n", "SELECT\n", @@ -339,29 +362,34 @@ " split_nitrites.nb_parametres,\n", " CASE\n", " WHEN\n", - " split_nitrites.valtraduite_no3 IS NULL \n", - " OR split_nitrites.valtraduite_no3_no2 IS NULL\n", - " OR split_nitrites.valtraduite_no3 = 0\n", - " OR split_nitrites.valtraduite_no3_no2 = 0\n", + " split_nitrites.nb_parametres != 3\n", + " OR (split_nitrites.valtraduite_no3 = 0\n", + " AND split_nitrites.valtraduite_no2 = 0\n", + " AND split_nitrites.valtraduite_no3_no2 = 0 )\n", " THEN 'non_quantifie'\n", " WHEN\n", - " split_nitrites.valtraduite_no3 <0.5\n", - " AND split_nitrites.valtraduite_no3_no2 <1\n", + " split_nitrites.nb_parametres = 3\n", + " AND split_nitrites.valtraduite_no3 < 50\n", + " AND split_nitrites.valtraduite_no2 < 0.5\n", + " AND split_nitrites.valtraduite_no3_no2 < 1\n", " THEN 'conforme'\n", " WHEN\n", - " split_nitrites.valtraduite_no3 >= 0.5\n", - " OR split_nitrites.valtraduite_no3_no2 >= 1\n", + " split_nitrites.nb_parametres = 3\n", + " AND ( split_nitrites.valtraduite_no3 >= 50\n", + " OR split_nitrites.valtraduite_no2 >= 0.5\n", + " OR split_nitrites.valtraduite_no3_no2 >= 1 )\n", " THEN 'non_conforme'\n", " ELSE 'error'\n", " END AS resultat\n", "FROM\n", " split_nitrites\n", + "\n", "\n" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "id": "2a287b57-fa15-4939-8209-7dcdf0e75eda", "metadata": {}, "outputs": [ @@ -380,7 +408,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4390b15f5dab4a1e9c25e33ede1ea674", + "model_id": "555cca18a88d479ea3e32b51bfd2d3d6", "version_major": 2, "version_minor": 0 }, @@ -416,7 +444,7 @@ "+----------+-----------+---------+-----------------------+---------------+----------+" ] }, - "execution_count": 7, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -431,18 +459,104 @@ " resultat='error'" ] }, + { + "cell_type": "code", + "execution_count": 5, + "id": "30487685-f619-43ab-be20-054cb46db3c6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f38172c713c7485b80f7ecf4cd4c1869", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
resultatmin_cdreseaumax_cdreseau
conforme001000003974004314
non_quantifie001000251976003554
non_conforme002000060095000346
" + ], + "text/plain": [ + "+---------------+--------------+--------------+\n", + "| resultat | min_cdreseau | max_cdreseau |\n", + "+---------------+--------------+--------------+\n", + "| conforme | 001000003 | 974004314 |\n", + "| non_quantifie | 001000251 | 976003554 |\n", + "| non_conforme | 002000060 | 095000346 |\n", + "+---------------+--------------+--------------+" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier\n", + "SELECT \n", + " resultat, MIN( cdreseau) min_cdreseau, MAX(cdreseau) AS max_cdreseau\n", + "FROM \n", + " int__resultats_nitrites_udi_dernier\n", + "GROUP BY 1" + ] + }, { "cell_type": "markdown", "id": "c596b48d-ebc4-4fb9-aab8-b626076792d2", "metadata": {}, "source": [ - "# Check errors" + "# Check " ] }, { "cell_type": "code", - "execution_count": 5, - "id": "2137dd22-ec22-4b37-93ee-30b75c06ad42", + "execution_count": 6, + "id": "761be547-f21e-4712-a898-4175f8faa89c", "metadata": {}, "outputs": [ { @@ -460,7 +574,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6b42c924850141e2b4be93ae13e43e86", + "model_id": "6ae97d2bf6e44b2facb25f8d40b6cfe7", "version_major": 2, "version_minor": 0 }, @@ -482,54 +596,130 @@ " nb_parametres\n", " dernier_prel_datetime\n", " valtraduite_no3\n", + " valtraduite_no2\n", " valtraduite_no3_no2\n", " \n", " \n", " \n", " \n", - " 973000028\n", + " 001000003\n", " nitrite\n", - " 2\n", - " 2024-12-02 11:30:00\n", + " 3\n", + " 2024-12-26 09:08:00\n", + " 13.000\n", " 0.000\n", - " None\n", + " 0.260\n", " \n", " \n", - " 088002296\n", + " 001000260\n", " nitrite\n", - " 1\n", - " 2024-09-27 12:00:00\n", - " None\n", - " None\n", + " 3\n", + " 2024-12-06 12:02:00\n", + " 2.400\n", + " 0.000\n", + " 0.050\n", + " \n", + " \n", + " 001000268\n", + " nitrite\n", + " 3\n", + " 2024-11-28 08:25:00\n", + " 2.600\n", + " 0.000\n", + " 0.050\n", + " \n", + " \n", + " 001000293\n", + " nitrite\n", + " 3\n", + " 2024-10-10 10:20:00\n", + " 4.000\n", + " 0.000\n", + " 0.080\n", + " \n", + " \n", + " 002000060\n", + " nitrite\n", + " 3\n", + " 2024-10-18 08:50:00\n", + " 59.300\n", + " 0.000\n", + " 0.000\n", " \n", " \n", " 073002059\n", " nitrite\n", " 1\n", " 2024-10-23 09:28:00\n", - " None\n", - " None\n", + " 0.000\n", + " 0.000\n", + " 0.000\n", + " \n", + " \n", + " 088002296\n", + " nitrite\n", + " 1\n", + " 2024-09-27 12:00:00\n", + " 0.000\n", + " 0.000\n", + " 0.000\n", + " \n", + " \n", + " 095000346\n", + " nitrite\n", + " 3\n", + " 2024-12-17 08:55:00\n", + " 51.000\n", + " 0.000\n", + " 1.000\n", + " \n", + " \n", + " 973000028\n", + " nitrite\n", + " 2\n", + " 2024-12-02 11:30:00\n", + " 0.000\n", + " 0.000\n", + " 0.000\n", + " \n", + " \n", + " 976003554\n", + " nitrite\n", + " 2\n", + " 2024-11-12 10:00:00\n", + " 0.000\n", + " 0.000\n", + " 0.000\n", " \n", " \n", - "" + "\n", + "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+---------------+-----------------------+-----------------+---------------------+\n", - "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | valtraduite_no3 | valtraduite_no3_no2 |\n", - "+-----------+-----------+---------------+-----------------------+-----------------+---------------------+\n", - "| 973000028 | nitrite | 2 | 2024-12-02 11:30:00 | 0.000 | None |\n", - "| 088002296 | nitrite | 1 | 2024-09-27 12:00:00 | None | None |\n", - "| 073002059 | nitrite | 1 | 2024-10-23 09:28:00 | None | None |\n", - "+-----------+-----------+---------------+-----------------------+-----------------+---------------------+" + "+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+\n", + "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | valtraduite_no3 | valtraduite_no2 | valtraduite_no3_no2 |\n", + "+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+\n", + "| 001000003 | nitrite | 3 | 2024-12-26 09:08:00 | 13.000 | 0.000 | 0.260 |\n", + "| 001000260 | nitrite | 3 | 2024-12-06 12:02:00 | 2.400 | 0.000 | 0.050 |\n", + "| 001000268 | nitrite | 3 | 2024-11-28 08:25:00 | 2.600 | 0.000 | 0.050 |\n", + "| 001000293 | nitrite | 3 | 2024-10-10 10:20:00 | 4.000 | 0.000 | 0.080 |\n", + "| 002000060 | nitrite | 3 | 2024-10-18 08:50:00 | 59.300 | 0.000 | 0.000 |\n", + "| 073002059 | nitrite | 1 | 2024-10-23 09:28:00 | 0.000 | 0.000 | 0.000 |\n", + "| 088002296 | nitrite | 1 | 2024-09-27 12:00:00 | 0.000 | 0.000 | 0.000 |\n", + "| 095000346 | nitrite | 3 | 2024-12-17 08:55:00 | 51.000 | 0.000 | 1.000 |\n", + "| 973000028 | nitrite | 2 | 2024-12-02 11:30:00 | 0.000 | 0.000 | 0.000 |\n", + "| 976003554 | nitrite | 2 | 2024-11-12 10:00:00 | 0.000 | 0.000 | 0.000 |\n", + "+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+\n", + "Truncated to displaylimit of 10." ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%sql --save int__resultats_nitrites_udi_dernier\n", + "%%sql\n", "WITH\n", "last_pvl AS (\n", " SELECT\n", @@ -549,6 +739,7 @@ " WHERE\n", " categorie = 'nitrite'\n", " AND\n", + " -- On garde les prélèvements de moins d'un an\n", " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", ")\n", " SELECT\n", @@ -556,22 +747,212 @@ " last_pvl.categorie,\n", " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", - " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3' THEN last_pvl.valtraduite END) AS valtraduite_no3,\n", - " MAX(CASE WHEN last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite END) AS valtraduite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN last_pvl.valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3_no2\n", " FROM\n", " last_pvl\n", " WHERE\n", " last_pvl.row_number = 1\n", - " AND cdreseau IN ('088002296','073002059','973000028')\n", - " GROUP BY \n", + " AND cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", + " GROUP BY\n", " last_pvl.cdreseau,\n", - " last_pvl.categorie \n" + " last_pvl.categorie\n", + " ORDER BY \n", + " cdreseau" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2137dd22-ec22-4b37-93ee-30b75c06ad42", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "64acfb690c854ae08912993e9f68d008", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000003nitritedernier_prel2024-12-26 09:08:003conforme
001000260nitritedernier_prel2024-12-06 12:02:003conforme
001000268nitritedernier_prel2024-11-28 08:25:003conforme
001000293nitritedernier_prel2024-10-10 10:20:003conforme
002000060nitritedernier_prel2024-10-18 08:50:003non_conforme
073002059nitritedernier_prel2024-10-23 09:28:001non_quantifie
088002296nitritedernier_prel2024-09-27 12:00:001non_quantifie
095000346nitritedernier_prel2024-12-17 08:55:003non_conforme
973000028nitritedernier_prel2024-12-02 11:30:002non_quantifie
976003554nitritedernier_prel2024-11-12 10:00:002non_quantifie
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "| 001000003 | nitrite | dernier_prel | 2024-12-26 09:08:00 | 3 | conforme |\n", + "| 001000260 | nitrite | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", + "| 001000268 | nitrite | dernier_prel | 2024-11-28 08:25:00 | 3 | conforme |\n", + "| 001000293 | nitrite | dernier_prel | 2024-10-10 10:20:00 | 3 | conforme |\n", + "| 002000060 | nitrite | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", + "| 073002059 | nitrite | dernier_prel | 2024-10-23 09:28:00 | 1 | non_quantifie |\n", + "| 088002296 | nitrite | dernier_prel | 2024-09-27 12:00:00 | 1 | non_quantifie |\n", + "| 095000346 | nitrite | dernier_prel | 2024-12-17 08:55:00 | 3 | non_conforme |\n", + "| 973000028 | nitrite | dernier_prel | 2024-12-02 11:30:00 | 2 | non_quantifie |\n", + "| 976003554 | nitrite | dernier_prel | 2024-11-12 10:00:00 | 2 | non_quantifie |\n", + "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier \n", + " SELECT \n", + " *\n", + " FROM\n", + " int__resultats_nitrites_udi_dernier\n", + "WHERE\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", + "ORDER BY \n", + " cdreseau" ] }, { "cell_type": "code", "execution_count": null, - "id": "761be547-f21e-4712-a898-4175f8faa89c", + "id": "a7ee43cb-a514-4d48-8e4d-8fa6d95fdec4", "metadata": {}, "outputs": [], "source": [] diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index 9811a967..a24ca37f 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -32,13 +32,23 @@ split_nitrites AS ( WHEN last_pvl.cdparametresiseeaux = 'NO3' THEN last_pvl.valtraduite + ELSE 0 END ) AS valtraduite_no3, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO2' + THEN last_pvl.valtraduite + ELSE 0 + END + ) AS valtraduite_no2, MAX( CASE WHEN last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite + ELSE 0 END ) AS valtraduite_no3_no2 FROM @@ -58,18 +68,26 @@ SELECT split_nitrites.nb_parametres, CASE WHEN - split_nitrites.valtraduite_no3 IS NULL - OR split_nitrites.valtraduite_no3_no2 IS NULL - OR split_nitrites.valtraduite_no3 = 0 - OR split_nitrites.valtraduite_no3_no2 = 0 + split_nitrites.nb_parametres != 3 + OR ( + split_nitrites.valtraduite_no3 = 0 + AND split_nitrites.valtraduite_no2 = 0 + AND split_nitrites.valtraduite_no3_no2 = 0 + ) THEN 'non_quantifie' WHEN - split_nitrites.valtraduite_no3 < 0.5 + split_nitrites.nb_parametres = 3 + AND split_nitrites.valtraduite_no3 < 50 + AND split_nitrites.valtraduite_no2 < 0.5 AND split_nitrites.valtraduite_no3_no2 < 1 THEN 'conforme' WHEN - split_nitrites.valtraduite_no3 >= 0.5 - OR split_nitrites.valtraduite_no3_no2 >= 1 + split_nitrites.nb_parametres = 3 + AND ( + split_nitrites.valtraduite_no3 >= 50 + OR split_nitrites.valtraduite_no2 >= 0.5 + OR split_nitrites.valtraduite_no3_no2 >= 1 + ) THEN 'non_conforme' ELSE 'error' END AS resultat From 75c93ccdead363c91ce1280bb0f766e1fc5cf47d Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sat, 5 Apr 2025 12:01:59 +0200 Subject: [PATCH 03/30] feat: add test for categorie nitrite --- .../intermediate/cvm/_int__cvm_models.yaml | 2 + dbt_/tests/test_nitrates_results.sql | 78 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 dbt_/tests/test_nitrates_results.sql diff --git a/dbt_/models/intermediate/cvm/_int__cvm_models.yaml b/dbt_/models/intermediate/cvm/_int__cvm_models.yaml index d022260c..a305fd85 100644 --- a/dbt_/models/intermediate/cvm/_int__cvm_models.yaml +++ b/dbt_/models/intermediate/cvm/_int__cvm_models.yaml @@ -27,7 +27,9 @@ models: - dbt_utils.expression_is_true: expression: "LIKE 'bilan_annuel%'" - name: nb_depassements + type: INT - name: nb_prelevements + type: INT - name: ratio_depassements description: "Pourcentage de prélèvements qui dépassent la limite" type: FLOAT diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql new file mode 100644 index 00000000..a8bbb2b5 --- /dev/null +++ b/dbt_/tests/test_nitrates_results.sql @@ -0,0 +1,78 @@ +-- dernier relevé +SELECT + 'dernier relevé' AS periode, + cdreseau, + categorie, + resultat, + 0 AS nb_depassements, + 0 AS nb_prelevements, + 0 AS ratio_depassements +FROM + {{ ref('int__resultats_nitrites_udi_dernier') }} +WHERE + ( + cdreseau = '001000003' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-12-26 09:08:00' + AND resultat != 'conforme' + ) + OR + ( + cdreseau = '001000260' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-12-06 12:02:00' + AND resultat != 'conforme' + ) + OR + ( + cdreseau = '001000268' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-11-28 08:25:00' + AND resultat != 'conforme' + ) + OR + ( + cdreseau = '002000060' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-10-18 08:50:00' + AND resultat != 'non_conforme' + ) + OR + ( + cdreseau = '073002059' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-10-23 09:28:00' + AND resultat != 'non_quantifie' + AND nb_parametres = 1 + ) + OR + ( + cdreseau = '088002296' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-09-27 12:00:00' + AND resultat != 'non_quantifie' + AND nb_parametres = 1 + ) + OR + ( + cdreseau = '095000346' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-12-17 08:55:00' + AND resultat != 'non_conforme' + ) + OR + ( + cdreseau = '973000028' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-12-17 08:55:00' + AND resultat != 'non_quantifie' + AND nb_parametres = 2 + ) + OR + ( + cdreseau = '976003554' + AND categorie = 'nitrite' + AND dernier_prel_datetime = '2024-11-12 10:00:00' + AND resultat != 'non_quantifie' + AND nb_parametres = 2 + ) From ad7e42a19d031cc2aec93b778d30d2d72caf15b2 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sat, 5 Apr 2025 21:51:08 +0200 Subject: [PATCH 04/30] feat: add last_prel nitrite into final table --- .../models/intermediate/int__union_resultats_udi.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index bfce23d9..73d66d2f 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -33,3 +33,15 @@ SELECT nb_parametres FROM {{ ref('int__resultats_cvm_udi_dernier') }} +UNION ALL +SELECT + cdreseau, + periode, + categorie, + resultat, + null AS ratio, + dernier_prel_datetime, + null AS dernier_prel_valeur, + nb_parametres +FROM + {{ ref('int__resultats_nitrites_udi_dernier') }} From 1c317a11238359abeef61999da5a41fbc27d5708 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sun, 6 Apr 2025 18:54:13 +0200 Subject: [PATCH 05/30] feat: add annuel nitrate --- .../intermediate/int__union_resultats_udi.sql | 12 +++ .../nitrites/_int__nitrites_models.yaml | 41 ++++++++- .../int__resultats_nitrites_udi_annuel.sql | 84 +++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index 73d66d2f..31bcb41d 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -45,3 +45,15 @@ SELECT nb_parametres FROM {{ ref('int__resultats_nitrites_udi_dernier') }} +UNION ALL +SELECT + cdreseau, + periode, + categorie, + null AS resultat, + ratio_depassements AS ratio, + null AS dernier_prel_datetime, + null AS dernier_prel_valeur, + null AS nb_parametres +FROM + {{ ref('int__resultats_nitrites_udi_annuel') }} diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index 261d349b..a8f61fc8 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -37,4 +37,43 @@ models: values: - non_quantifie - conforme - - non_conforme \ No newline at end of file + - non_conforme + + - name: int__resultats_nitrites_udi_annuel + description: Résultats nitrites par UDI pour la temporalité 'bilan_annuel' + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cdreseau + - annee + columns: + - name: cdreseau + description: "Code de l'installation (unité de distribution)" + tests: + - not_null + - name: annee + description: "Annee du prevelvement" + type: SMALLINT + - name: categorie + description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + tests: + - accepted_values: + values: ["nitrite"] + - name: periode + description: "Période du résultat" + type: VARCHAR + tests: + - dbt_utils.expression_is_true: + expression: "LIKE 'bilan_annuel%'" + - name: nb_depassements + type: INT + - name: nb_prelevements + type: INT + - name: ratio_depassements + description: "Pourcentage de prélèvements qui dépassent la limite" + type: FLOAT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql new file mode 100644 index 00000000..052fc8a4 --- /dev/null +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql @@ -0,0 +1,84 @@ +WITH +prels AS ( + SELECT + de_partition AS annee, + cdreseau, + referenceprel, + datetimeprel, + COUNT(DISTINCT cdparametresiseeaux) AS nb_parametres, + MAX( + CASE + WHEN + cdparametresiseeaux = 'NO3' + THEN valtraduite + ELSE 0 + END + ) AS valtraduite_no3, + MAX( + CASE + WHEN + cdparametresiseeaux = 'NO2' + THEN valtraduite + ELSE 0 + END + ) AS valtraduite_no2, + MAX( + CASE + WHEN + cdparametresiseeaux = 'NO3_NO2' + THEN valtraduite + ELSE 0 + END + ) AS valtraduite_no3_no2 + FROM + {{ ref('int__resultats_udi_communes') }} + WHERE + categorie = 'nitrite' + GROUP BY + annee, + cdreseau, + referenceprel, + datetimeprel, +) + + +SELECT + prels.cdreseau, + prels.annee, + 'nitrite' AS categorie, + 'bilan_annuel_' || prels.annee AS periode, + COUNT( + DISTINCT + CASE + WHEN + prels.nb_parametres = 3 + AND ( + prels.valtraduite_no3 >= 50 + OR prels.valtraduite_no2 >= 0.5 + OR prels.valtraduite_no3_no2 >= 1 + ) + THEN prels.referenceprel + END + ) AS nb_depassements, + COUNT(DISTINCT prels.referenceprel) AS nb_prelevements, + ( + COUNT( + DISTINCT + CASE + WHEN + prels.nb_parametres = 3 + AND ( + prels.valtraduite_no3 >= 50 + OR prels.valtraduite_no2 >= 0.5 + OR prels.valtraduite_no3_no2 >= 1 + ) + THEN prels.referenceprel + END + )::float + / + COUNT(DISTINCT prels.referenceprel)::float + ) AS ratio_depassements + +FROM prels + +GROUP BY prels.cdreseau, prels.annee From ffa66d621cb14acf031ea5c9c05c9642cd391757 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sun, 6 Apr 2025 19:00:25 +0200 Subject: [PATCH 06/30] fix: update CASE WHEN for resultat --- .../int__resultats_nitrites_udi_dernier.sql | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index a24ca37f..14a71d99 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -68,12 +68,9 @@ SELECT split_nitrites.nb_parametres, CASE WHEN - split_nitrites.nb_parametres != 3 - OR ( - split_nitrites.valtraduite_no3 = 0 - AND split_nitrites.valtraduite_no2 = 0 - AND split_nitrites.valtraduite_no3_no2 = 0 - ) + split_nitrites.valtraduite_no3 = 0 + AND split_nitrites.valtraduite_no2 = 0 + AND split_nitrites.valtraduite_no3_no2 = 0 THEN 'non_quantifie' WHEN split_nitrites.nb_parametres = 3 @@ -82,13 +79,18 @@ SELECT AND split_nitrites.valtraduite_no3_no2 < 1 THEN 'conforme' WHEN - split_nitrites.nb_parametres = 3 + split_nitrites.valtraduite_no3 >= 50 + OR split_nitrites.valtraduite_no2 >= 0.5 + OR split_nitrites.valtraduite_no3_no2 >= 1 + THEN 'non_conforme' + WHEN + split_nitrites.nb_parametres != 3 AND ( - split_nitrites.valtraduite_no3 >= 50 - OR split_nitrites.valtraduite_no2 >= 0.5 - OR split_nitrites.valtraduite_no3_no2 >= 1 + split_nitrites.valtraduite_no3 < 50 + OR split_nitrites.valtraduite_no2 < 0.5 + OR split_nitrites.valtraduite_no3_no2 < 1 ) - THEN 'non_conforme' + THEN 'non_quantifie' ELSE 'error' END AS resultat FROM From 1dde09b594a5b196c6932ffbe6497872d0fc87c8 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sun, 6 Apr 2025 19:15:42 +0200 Subject: [PATCH 07/30] fix: update CASE WHEN for resultat and nb_depassements --- .../int__resultats_nitrites_udi_annuel.sql | 18 +++++--------- .../int__resultats_nitrites_udi_dernier.sql | 24 ++++++++++--------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql index 052fc8a4..891b378a 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql @@ -51,12 +51,9 @@ SELECT DISTINCT CASE WHEN - prels.nb_parametres = 3 - AND ( - prels.valtraduite_no3 >= 50 - OR prels.valtraduite_no2 >= 0.5 - OR prels.valtraduite_no3_no2 >= 1 - ) + prels.valtraduite_no3 >= 50 + OR prels.valtraduite_no2 >= 0.5 + OR prels.valtraduite_no3_no2 >= 1 THEN prels.referenceprel END ) AS nb_depassements, @@ -66,12 +63,9 @@ SELECT DISTINCT CASE WHEN - prels.nb_parametres = 3 - AND ( - prels.valtraduite_no3 >= 50 - OR prels.valtraduite_no2 >= 0.5 - OR prels.valtraduite_no3_no2 >= 1 - ) + prels.valtraduite_no3 >= 50 + OR prels.valtraduite_no2 >= 0.5 + OR prels.valtraduite_no3_no2 >= 1 THEN prels.referenceprel END )::float diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index a24ca37f..14a71d99 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -68,12 +68,9 @@ SELECT split_nitrites.nb_parametres, CASE WHEN - split_nitrites.nb_parametres != 3 - OR ( - split_nitrites.valtraduite_no3 = 0 - AND split_nitrites.valtraduite_no2 = 0 - AND split_nitrites.valtraduite_no3_no2 = 0 - ) + split_nitrites.valtraduite_no3 = 0 + AND split_nitrites.valtraduite_no2 = 0 + AND split_nitrites.valtraduite_no3_no2 = 0 THEN 'non_quantifie' WHEN split_nitrites.nb_parametres = 3 @@ -82,13 +79,18 @@ SELECT AND split_nitrites.valtraduite_no3_no2 < 1 THEN 'conforme' WHEN - split_nitrites.nb_parametres = 3 + split_nitrites.valtraduite_no3 >= 50 + OR split_nitrites.valtraduite_no2 >= 0.5 + OR split_nitrites.valtraduite_no3_no2 >= 1 + THEN 'non_conforme' + WHEN + split_nitrites.nb_parametres != 3 AND ( - split_nitrites.valtraduite_no3 >= 50 - OR split_nitrites.valtraduite_no2 >= 0.5 - OR split_nitrites.valtraduite_no3_no2 >= 1 + split_nitrites.valtraduite_no3 < 50 + OR split_nitrites.valtraduite_no2 < 0.5 + OR split_nitrites.valtraduite_no3_no2 < 1 ) - THEN 'non_conforme' + THEN 'non_quantifie' ELSE 'error' END AS resultat FROM From cdc3dee942928022a1d1053c298a604cab4a4e31 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:32:53 +0200 Subject: [PATCH 08/30] feat: update possible values into resultat --- analytics/notebooks/resultats_nitrates.ipynb | 321 ++++++++++++------ .../nitrites/_int__nitrites_models.yaml | 2 +- .../int__resultats_nitrites_udi_dernier.sql | 58 +++- dbt_/tests/test_nitrates_results.sql | 10 +- 4 files changed, 263 insertions(+), 128 deletions(-) diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb index ee16eafe..ac281159 100644 --- a/analytics/notebooks/resultats_nitrates.ipynb +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -157,7 +157,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5d6d76c002514968b669f2488a28e708", + "model_id": "393adae3495f4ce48ef2f5c79b4a58f1", "version_major": 2, "version_minor": 0 }, @@ -184,82 +184,82 @@ " \n", " \n", " \n", - " 001000288\n", + " 038001345\n", " nitrite\n", " dernier_prel\n", - " 2024-11-27 11:52:00\n", + " 2024-12-03 09:10:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000330\n", + " 038001363\n", " nitrite\n", " dernier_prel\n", - " 2024-12-13 08:49:00\n", + " 2024-10-17 10:15:00\n", " 3\n", - " conforme\n", + " donnee_manquante\n", " \n", " \n", - " 001000337\n", + " 038001404\n", " nitrite\n", " dernier_prel\n", - " 2024-07-04 08:31:00\n", + " 2024-09-13 08:05:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000375\n", + " 038001576\n", " nitrite\n", " dernier_prel\n", - " 2024-08-28 10:11:00\n", + " 2024-10-15 10:10:00\n", " 3\n", - " conforme\n", + " donnee_manquante\n", " \n", " \n", - " 001000459\n", + " 038001625\n", " nitrite\n", " dernier_prel\n", - " 2024-07-11 10:51:00\n", + " 2024-12-17 10:25:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000526\n", + " 038001899\n", " nitrite\n", " dernier_prel\n", - " 2024-08-05 08:39:00\n", + " 2024-11-13 07:35:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000592\n", + " 038001960\n", " nitrite\n", " dernier_prel\n", - " 2024-11-28 10:22:00\n", + " 2024-10-14 11:25:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000607\n", + " 038002035\n", " nitrite\n", " dernier_prel\n", - " 2024-12-09 08:11:00\n", + " 2024-10-09 08:55:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000814\n", + " 038002151\n", " nitrite\n", " dernier_prel\n", - " 2024-12-30 11:01:00\n", + " 2024-11-18 10:25:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000823\n", + " 038002176\n", " nitrite\n", " dernier_prel\n", - " 2024-10-18 09:24:00\n", + " 2024-10-07 06:50:00\n", " 3\n", " conforme\n", " \n", @@ -268,20 +268,20 @@ "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", - "| 001000288 | nitrite | dernier_prel | 2024-11-27 11:52:00 | 3 | conforme |\n", - "| 001000330 | nitrite | dernier_prel | 2024-12-13 08:49:00 | 3 | conforme |\n", - "| 001000337 | nitrite | dernier_prel | 2024-07-04 08:31:00 | 3 | conforme |\n", - "| 001000375 | nitrite | dernier_prel | 2024-08-28 10:11:00 | 3 | conforme |\n", - "| 001000459 | nitrite | dernier_prel | 2024-07-11 10:51:00 | 3 | conforme |\n", - "| 001000526 | nitrite | dernier_prel | 2024-08-05 08:39:00 | 3 | conforme |\n", - "| 001000592 | nitrite | dernier_prel | 2024-11-28 10:22:00 | 3 | conforme |\n", - "| 001000607 | nitrite | dernier_prel | 2024-12-09 08:11:00 | 3 | conforme |\n", - "| 001000814 | nitrite | dernier_prel | 2024-12-30 11:01:00 | 3 | conforme |\n", - "| 001000823 | nitrite | dernier_prel | 2024-10-18 09:24:00 | 3 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| 038001345 | nitrite | dernier_prel | 2024-12-03 09:10:00 | 3 | conforme |\n", + "| 038001363 | nitrite | dernier_prel | 2024-10-17 10:15:00 | 3 | donnee_manquante |\n", + "| 038001404 | nitrite | dernier_prel | 2024-09-13 08:05:00 | 3 | conforme |\n", + "| 038001576 | nitrite | dernier_prel | 2024-10-15 10:10:00 | 3 | donnee_manquante |\n", + "| 038001625 | nitrite | dernier_prel | 2024-12-17 10:25:00 | 3 | conforme |\n", + "| 038001899 | nitrite | dernier_prel | 2024-11-13 07:35:00 | 3 | conforme |\n", + "| 038001960 | nitrite | dernier_prel | 2024-10-14 11:25:00 | 3 | conforme |\n", + "| 038002035 | nitrite | dernier_prel | 2024-10-09 08:55:00 | 3 | conforme |\n", + "| 038002151 | nitrite | dernier_prel | 2024-11-18 10:25:00 | 3 | conforme |\n", + "| 038002176 | nitrite | dernier_prel | 2024-10-07 06:50:00 | 3 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", "Truncated to displaylimit of 10." ] }, @@ -325,8 +325,28 @@ " CASE\n", " WHEN\n", " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n03,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n02,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", " THEN last_pvl.valtraduite\n", - " ELSE 0\n", " END\n", " ) AS valtraduite_no3,\n", " MAX(\n", @@ -334,7 +354,6 @@ " WHEN\n", " last_pvl.cdparametresiseeaux = 'NO2'\n", " THEN last_pvl.valtraduite\n", - " ELSE 0\n", " END\n", " ) AS valtraduite_no2,\n", " MAX(\n", @@ -342,7 +361,6 @@ " WHEN\n", " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", " THEN last_pvl.valtraduite\n", - " ELSE 0\n", " END\n", " ) AS valtraduite_no3_no2\n", " FROM\n", @@ -362,11 +380,32 @@ " split_nitrites.nb_parametres,\n", " CASE\n", " WHEN\n", - " split_nitrites.nb_parametres != 3\n", - " OR (split_nitrites.valtraduite_no3 = 0\n", - " AND split_nitrites.valtraduite_no2 = 0\n", - " AND split_nitrites.valtraduite_no3_no2 = 0 )\n", - " THEN 'non_quantifie'\n", + " DATE_DIFF(\n", + " 'day',\n", + " LEAST(\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02,\n", + " split_nitrites.dernier_prel_datetime_no3_no2\n", + " ),\n", + " GREATEST(\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02,\n", + " split_nitrites.dernier_prel_datetime_no3_no2\n", + " )\n", + " ) >= 30\n", + " OR split_nitrites.valtraduite_no2 IS NULL\n", + " OR split_nitrites.valtraduite_no3 IS NULL\n", + " THEN 'donnee_manquante'\n", + " WHEN\n", + " split_nitrites.valtraduite_no3_no2 IS NULL\n", + " AND split_nitrites.valtraduite_no2 IS NOT NULL\n", + " AND split_nitrites.valtraduite_no3 IS NOT NULL\n", + " AND split_nitrites.valtraduite_no3 < 50\n", + " AND split_nitrites.valtraduite_no2 < 0.5\n", + " AND split_nitrites.valtraduite_no3 / 50\n", + " + split_nitrites.valtraduite_no2 / 3\n", + " < 1\n", + " THEN 'conforme'\n", " WHEN\n", " split_nitrites.nb_parametres = 3\n", " AND split_nitrites.valtraduite_no3 < 50\n", @@ -374,17 +413,22 @@ " AND split_nitrites.valtraduite_no3_no2 < 1\n", " THEN 'conforme'\n", " WHEN\n", - " split_nitrites.nb_parametres = 3\n", - " AND ( split_nitrites.valtraduite_no3 >= 50\n", + " split_nitrites.valtraduite_no3 >= 50\n", " OR split_nitrites.valtraduite_no2 >= 0.5\n", - " OR split_nitrites.valtraduite_no3_no2 >= 1 )\n", + " OR split_nitrites.valtraduite_no3_no2 >= 1\n", " THEN 'non_conforme'\n", + " /* WHEN\n", + " split_nitrites.nb_parametres != 3\n", + " AND (\n", + " split_nitrites.valtraduite_no3 < 50\n", + " OR split_nitrites.valtraduite_no2 < 0.5\n", + " OR split_nitrites.valtraduite_no3_no2 < 1\n", + " )\n", + " THEN 'non_quantifie'*/\n", " ELSE 'error'\n", " END AS resultat\n", "FROM\n", - " split_nitrites\n", - "\n", - "\n" + " split_nitrites\n" ] }, { @@ -408,7 +452,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "555cca18a88d479ea3e32b51bfd2d3d6", + "model_id": "929b4592e19847a2802cb22edcf32040", "version_major": 2, "version_minor": 0 }, @@ -480,7 +524,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f38172c713c7485b80f7ecf4cd4c1869", + "model_id": "4c142c15520f4e5a9e4d583196cc29e4", "version_major": 2, "version_minor": 0 }, @@ -504,31 +548,31 @@ " \n", " \n", " \n", + " non_conforme\n", + " 002000060\n", + " 095000307\n", + " \n", + " \n", " conforme\n", " 001000003\n", - " 974004314\n", + " 976003487\n", " \n", " \n", - " non_quantifie\n", + " donnee_manquante\n", " 001000251\n", " 976003554\n", " \n", - " \n", - " non_conforme\n", - " 002000060\n", - " 095000346\n", - " \n", " \n", "" ], "text/plain": [ - "+---------------+--------------+--------------+\n", - "| resultat | min_cdreseau | max_cdreseau |\n", - "+---------------+--------------+--------------+\n", - "| conforme | 001000003 | 974004314 |\n", - "| non_quantifie | 001000251 | 976003554 |\n", - "| non_conforme | 002000060 | 095000346 |\n", - "+---------------+--------------+--------------+" + "+------------------+--------------+--------------+\n", + "| resultat | min_cdreseau | max_cdreseau |\n", + "+------------------+--------------+--------------+\n", + "| non_conforme | 002000060 | 095000307 |\n", + "| conforme | 001000003 | 976003487 |\n", + "| donnee_manquante | 001000251 | 976003554 |\n", + "+------------------+--------------+--------------+" ] }, "execution_count": 5, @@ -555,7 +599,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "761be547-f21e-4712-a898-4175f8faa89c", "metadata": {}, "outputs": [ @@ -574,7 +618,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6ae97d2bf6e44b2facb25f8d40b6cfe7", + "model_id": "08da46f805e6474bb69cb35f30140197", "version_major": 2, "version_minor": 0 }, @@ -595,6 +639,9 @@ " categorie\n", " nb_parametres\n", " dernier_prel_datetime\n", + " dernier_prel_datetime_n03\n", + " dernier_prel_datetime_n02\n", + " dernier_prel_datetime_no3_no2\n", " valtraduite_no3\n", " valtraduite_no2\n", " valtraduite_no3_no2\n", @@ -606,6 +653,9 @@ " nitrite\n", " 3\n", " 2024-12-26 09:08:00\n", + " 2024-12-26 09:08:00\n", + " 2024-12-26 09:08:00\n", + " 2024-12-26 09:08:00\n", " 13.000\n", " 0.000\n", " 0.260\n", @@ -615,6 +665,9 @@ " nitrite\n", " 3\n", " 2024-12-06 12:02:00\n", + " 2024-12-06 12:02:00\n", + " 2024-12-06 12:02:00\n", + " 2024-12-06 12:02:00\n", " 2.400\n", " 0.000\n", " 0.050\n", @@ -624,6 +677,9 @@ " nitrite\n", " 3\n", " 2024-11-28 08:25:00\n", + " 2024-11-28 08:25:00\n", + " 2024-11-28 08:25:00\n", + " 2024-11-28 08:25:00\n", " 2.600\n", " 0.000\n", " 0.050\n", @@ -633,6 +689,9 @@ " nitrite\n", " 3\n", " 2024-10-10 10:20:00\n", + " 2024-10-10 10:20:00\n", + " 2024-10-10 10:20:00\n", + " 2024-10-10 10:20:00\n", " 4.000\n", " 0.000\n", " 0.080\n", @@ -642,6 +701,9 @@ " nitrite\n", " 3\n", " 2024-10-18 08:50:00\n", + " 2024-10-18 08:50:00\n", + " 2024-10-18 08:50:00\n", + " 2024-10-18 08:31:00\n", " 59.300\n", " 0.000\n", " 0.000\n", @@ -651,24 +713,33 @@ " nitrite\n", " 1\n", " 2024-10-23 09:28:00\n", + " None\n", + " 2024-10-23 09:28:00\n", + " None\n", + " None\n", " 0.000\n", - " 0.000\n", - " 0.000\n", + " None\n", " \n", " \n", " 088002296\n", " nitrite\n", " 1\n", " 2024-09-27 12:00:00\n", + " None\n", + " 2024-09-27 12:00:00\n", + " None\n", + " None\n", " 0.000\n", - " 0.000\n", - " 0.000\n", + " None\n", " \n", " \n", " 095000346\n", " nitrite\n", " 3\n", " 2024-12-17 08:55:00\n", + " 2024-12-17 08:55:00\n", + " 2024-07-01 09:20:00\n", + " 2024-07-01 09:20:00\n", " 51.000\n", " 0.000\n", " 1.000\n", @@ -678,42 +749,48 @@ " nitrite\n", " 2\n", " 2024-12-02 11:30:00\n", + " 2024-12-02 11:30:00\n", + " 2024-12-02 11:30:00\n", + " None\n", " 0.000\n", " 0.000\n", - " 0.000\n", + " None\n", " \n", " \n", " 976003554\n", " nitrite\n", " 2\n", " 2024-11-12 10:00:00\n", + " 2024-11-12 10:00:00\n", + " 2024-09-10 09:20:00\n", + " None\n", " 0.000\n", " 0.000\n", - " 0.000\n", + " None\n", " \n", " \n", "\n", "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+\n", - "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | valtraduite_no3 | valtraduite_no2 | valtraduite_no3_no2 |\n", - "+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+\n", - "| 001000003 | nitrite | 3 | 2024-12-26 09:08:00 | 13.000 | 0.000 | 0.260 |\n", - "| 001000260 | nitrite | 3 | 2024-12-06 12:02:00 | 2.400 | 0.000 | 0.050 |\n", - "| 001000268 | nitrite | 3 | 2024-11-28 08:25:00 | 2.600 | 0.000 | 0.050 |\n", - "| 001000293 | nitrite | 3 | 2024-10-10 10:20:00 | 4.000 | 0.000 | 0.080 |\n", - "| 002000060 | nitrite | 3 | 2024-10-18 08:50:00 | 59.300 | 0.000 | 0.000 |\n", - "| 073002059 | nitrite | 1 | 2024-10-23 09:28:00 | 0.000 | 0.000 | 0.000 |\n", - "| 088002296 | nitrite | 1 | 2024-09-27 12:00:00 | 0.000 | 0.000 | 0.000 |\n", - "| 095000346 | nitrite | 3 | 2024-12-17 08:55:00 | 51.000 | 0.000 | 1.000 |\n", - "| 973000028 | nitrite | 2 | 2024-12-02 11:30:00 | 0.000 | 0.000 | 0.000 |\n", - "| 976003554 | nitrite | 2 | 2024-11-12 10:00:00 | 0.000 | 0.000 | 0.000 |\n", - "+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+\n", + "+-----------+-----------+---------------+-----------------------+---------------------------+---------------------------+-------------------------------+-----------------+-----------------+---------------------+\n", + "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | dernier_prel_datetime_n03 | dernier_prel_datetime_n02 | dernier_prel_datetime_no3_no2 | valtraduite_no3 | valtraduite_no2 | valtraduite_no3_no2 |\n", + "+-----------+-----------+---------------+-----------------------+---------------------------+---------------------------+-------------------------------+-----------------+-----------------+---------------------+\n", + "| 001000003 | nitrite | 3 | 2024-12-26 09:08:00 | 2024-12-26 09:08:00 | 2024-12-26 09:08:00 | 2024-12-26 09:08:00 | 13.000 | 0.000 | 0.260 |\n", + "| 001000260 | nitrite | 3 | 2024-12-06 12:02:00 | 2024-12-06 12:02:00 | 2024-12-06 12:02:00 | 2024-12-06 12:02:00 | 2.400 | 0.000 | 0.050 |\n", + "| 001000268 | nitrite | 3 | 2024-11-28 08:25:00 | 2024-11-28 08:25:00 | 2024-11-28 08:25:00 | 2024-11-28 08:25:00 | 2.600 | 0.000 | 0.050 |\n", + "| 001000293 | nitrite | 3 | 2024-10-10 10:20:00 | 2024-10-10 10:20:00 | 2024-10-10 10:20:00 | 2024-10-10 10:20:00 | 4.000 | 0.000 | 0.080 |\n", + "| 002000060 | nitrite | 3 | 2024-10-18 08:50:00 | 2024-10-18 08:50:00 | 2024-10-18 08:50:00 | 2024-10-18 08:31:00 | 59.300 | 0.000 | 0.000 |\n", + "| 073002059 | nitrite | 1 | 2024-10-23 09:28:00 | None | 2024-10-23 09:28:00 | None | None | 0.000 | None |\n", + "| 088002296 | nitrite | 1 | 2024-09-27 12:00:00 | None | 2024-09-27 12:00:00 | None | None | 0.000 | None |\n", + "| 095000346 | nitrite | 3 | 2024-12-17 08:55:00 | 2024-12-17 08:55:00 | 2024-07-01 09:20:00 | 2024-07-01 09:20:00 | 51.000 | 0.000 | 1.000 |\n", + "| 973000028 | nitrite | 2 | 2024-12-02 11:30:00 | 2024-12-02 11:30:00 | 2024-12-02 11:30:00 | None | 0.000 | 0.000 | None |\n", + "| 976003554 | nitrite | 2 | 2024-11-12 10:00:00 | 2024-11-12 10:00:00 | 2024-09-10 09:20:00 | None | 0.000 | 0.000 | None |\n", + "+-----------+-----------+---------------+-----------------------+---------------------------+---------------------------+-------------------------------+-----------------+-----------------+---------------------+\n", "Truncated to displaylimit of 10." ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -751,8 +828,28 @@ " CASE\n", " WHEN\n", " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n03,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n02,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", " THEN last_pvl.valtraduite\n", - " ELSE 0\n", " END\n", " ) AS valtraduite_no3,\n", " MAX(\n", @@ -760,7 +857,6 @@ " WHEN\n", " last_pvl.cdparametresiseeaux = 'NO2'\n", " THEN last_pvl.valtraduite\n", - " ELSE 0\n", " END\n", " ) AS valtraduite_no2,\n", " MAX(\n", @@ -768,7 +864,6 @@ " WHEN\n", " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", " THEN last_pvl.valtraduite\n", - " ELSE 0\n", " END\n", " ) AS valtraduite_no3_no2\n", " FROM\n", @@ -804,7 +899,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "64acfb690c854ae08912993e9f68d008", + "model_id": "f39eb246f635494e933f46977dd4b3d9", "version_major": 2, "version_minor": 0 }, @@ -876,7 +971,7 @@ " dernier_prel\n", " 2024-10-23 09:28:00\n", " 1\n", - " non_quantifie\n", + " donnee_manquante\n", " \n", " \n", " 088002296\n", @@ -884,7 +979,7 @@ " dernier_prel\n", " 2024-09-27 12:00:00\n", " 1\n", - " non_quantifie\n", + " donnee_manquante\n", " \n", " \n", " 095000346\n", @@ -892,7 +987,7 @@ " dernier_prel\n", " 2024-12-17 08:55:00\n", " 3\n", - " non_conforme\n", + " donnee_manquante\n", " \n", " \n", " 973000028\n", @@ -900,7 +995,7 @@ " dernier_prel\n", " 2024-12-02 11:30:00\n", " 2\n", - " non_quantifie\n", + " conforme\n", " \n", " \n", " 976003554\n", @@ -908,27 +1003,27 @@ " dernier_prel\n", " 2024-11-12 10:00:00\n", " 2\n", - " non_quantifie\n", + " donnee_manquante\n", " \n", " \n", "\n", "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", - "| 001000003 | nitrite | dernier_prel | 2024-12-26 09:08:00 | 3 | conforme |\n", - "| 001000260 | nitrite | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", - "| 001000268 | nitrite | dernier_prel | 2024-11-28 08:25:00 | 3 | conforme |\n", - "| 001000293 | nitrite | dernier_prel | 2024-10-10 10:20:00 | 3 | conforme |\n", - "| 002000060 | nitrite | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", - "| 073002059 | nitrite | dernier_prel | 2024-10-23 09:28:00 | 1 | non_quantifie |\n", - "| 088002296 | nitrite | dernier_prel | 2024-09-27 12:00:00 | 1 | non_quantifie |\n", - "| 095000346 | nitrite | dernier_prel | 2024-12-17 08:55:00 | 3 | non_conforme |\n", - "| 973000028 | nitrite | dernier_prel | 2024-12-02 11:30:00 | 2 | non_quantifie |\n", - "| 976003554 | nitrite | dernier_prel | 2024-11-12 10:00:00 | 2 | non_quantifie |\n", - "+-----------+-----------+--------------+-----------------------+---------------+---------------+\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| 001000003 | nitrite | dernier_prel | 2024-12-26 09:08:00 | 3 | conforme |\n", + "| 001000260 | nitrite | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", + "| 001000268 | nitrite | dernier_prel | 2024-11-28 08:25:00 | 3 | conforme |\n", + "| 001000293 | nitrite | dernier_prel | 2024-10-10 10:20:00 | 3 | conforme |\n", + "| 002000060 | nitrite | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", + "| 073002059 | nitrite | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", + "| 088002296 | nitrite | dernier_prel | 2024-09-27 12:00:00 | 1 | donnee_manquante |\n", + "| 095000346 | nitrite | dernier_prel | 2024-12-17 08:55:00 | 3 | donnee_manquante |\n", + "| 973000028 | nitrite | dernier_prel | 2024-12-02 11:30:00 | 2 | conforme |\n", + "| 976003554 | nitrite | dernier_prel | 2024-11-12 10:00:00 | 2 | donnee_manquante |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", "Truncated to displaylimit of 10." ] }, diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index 261d349b..c0ea60e9 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -35,6 +35,6 @@ models: - not_null - accepted_values: values: - - non_quantifie + - donnee_manquante - conforme - non_conforme \ No newline at end of file diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index 14a71d99..5c8ebb90 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -27,12 +27,32 @@ split_nitrites AS ( last_pvl.categorie, COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres, MAX(last_pvl.datetimeprel) AS dernier_prel_datetime, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3' + THEN last_pvl.datetimeprel + END + ) AS dernier_prel_datetime_n03, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO2' + THEN last_pvl.datetimeprel + END + ) AS dernier_prel_datetime_n02, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3_NO2' + THEN last_pvl.datetimeprel + END + ) AS dernier_prel_datetime_no3_no2, MAX( CASE WHEN last_pvl.cdparametresiseeaux = 'NO3' THEN last_pvl.valtraduite - ELSE 0 END ) AS valtraduite_no3, MAX( @@ -40,7 +60,6 @@ split_nitrites AS ( WHEN last_pvl.cdparametresiseeaux = 'NO2' THEN last_pvl.valtraduite - ELSE 0 END ) AS valtraduite_no2, MAX( @@ -48,7 +67,6 @@ split_nitrites AS ( WHEN last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite - ELSE 0 END ) AS valtraduite_no3_no2 FROM @@ -68,10 +86,32 @@ SELECT split_nitrites.nb_parametres, CASE WHEN - split_nitrites.valtraduite_no3 = 0 - AND split_nitrites.valtraduite_no2 = 0 - AND split_nitrites.valtraduite_no3_no2 = 0 - THEN 'non_quantifie' + DATE_DIFF( + 'day', + LEAST( + split_nitrites.dernier_prel_datetime_n03, + split_nitrites.dernier_prel_datetime_n02, + split_nitrites.dernier_prel_datetime_no3_no2 + ), + GREATEST( + split_nitrites.dernier_prel_datetime_n03, + split_nitrites.dernier_prel_datetime_n02, + split_nitrites.dernier_prel_datetime_no3_no2 + ) + ) >= 30 + OR split_nitrites.valtraduite_no2 IS NULL + OR split_nitrites.valtraduite_no3 IS NULL + THEN 'donnee_manquante' + WHEN + split_nitrites.valtraduite_no3_no2 IS NULL + AND split_nitrites.valtraduite_no2 IS NOT NULL + AND split_nitrites.valtraduite_no3 IS NOT NULL + AND split_nitrites.valtraduite_no3 < 50 + AND split_nitrites.valtraduite_no2 < 0.5 + AND split_nitrites.valtraduite_no3 / 50 + + split_nitrites.valtraduite_no2 / 3 + < 1 + THEN 'conforme' WHEN split_nitrites.nb_parametres = 3 AND split_nitrites.valtraduite_no3 < 50 @@ -83,14 +123,14 @@ SELECT OR split_nitrites.valtraduite_no2 >= 0.5 OR split_nitrites.valtraduite_no3_no2 >= 1 THEN 'non_conforme' - WHEN + /* WHEN split_nitrites.nb_parametres != 3 AND ( split_nitrites.valtraduite_no3 < 50 OR split_nitrites.valtraduite_no2 < 0.5 OR split_nitrites.valtraduite_no3_no2 < 1 ) - THEN 'non_quantifie' + THEN 'non_quantifie'*/ ELSE 'error' END AS resultat FROM diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index a8bbb2b5..f9d2156c 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -42,7 +42,7 @@ WHERE cdreseau = '073002059' AND categorie = 'nitrite' AND dernier_prel_datetime = '2024-10-23 09:28:00' - AND resultat != 'non_quantifie' + AND resultat != 'donnee_manquante' AND nb_parametres = 1 ) OR @@ -50,7 +50,7 @@ WHERE cdreseau = '088002296' AND categorie = 'nitrite' AND dernier_prel_datetime = '2024-09-27 12:00:00' - AND resultat != 'non_quantifie' + AND resultat != 'donnee_manquante' AND nb_parametres = 1 ) OR @@ -58,14 +58,14 @@ WHERE cdreseau = '095000346' AND categorie = 'nitrite' AND dernier_prel_datetime = '2024-12-17 08:55:00' - AND resultat != 'non_conforme' + AND resultat != 'donnee_manquante' ) OR ( cdreseau = '973000028' AND categorie = 'nitrite' AND dernier_prel_datetime = '2024-12-17 08:55:00' - AND resultat != 'non_quantifie' + AND resultat != 'conforme' AND nb_parametres = 2 ) OR @@ -73,6 +73,6 @@ WHERE cdreseau = '976003554' AND categorie = 'nitrite' AND dernier_prel_datetime = '2024-11-12 10:00:00' - AND resultat != 'non_quantifie' + AND resultat != 'donnee_manquante' AND nb_parametres = 2 ) From 54654d9dbc972e2da5f314bfb2a1adbcf9425cbf Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 11 Apr 2025 16:21:17 +0200 Subject: [PATCH 09/30] feat: use new file GF --- .../nitrites/_int__nitrites_models.yaml | 4 +- .../int__resultats_nitrites_udi_annuel.sql | 42 +++++++++++--- .../int__resultats_nitrites_udi_dernier.sql | 55 +++++++++++++++---- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index a8f61fc8..01311d82 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -12,7 +12,7 @@ models: tests: - not_null - accepted_values: - values: ["nitrite"] + values: ["nitrate"] - name: periode type: VARCHAR tests: @@ -58,7 +58,7 @@ models: description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." tests: - accepted_values: - values: ["nitrite"] + values: ["nitrate"] - name: periode description: "Période du résultat" type: VARCHAR diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql index 891b378a..fc0d6c90 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql @@ -29,11 +29,35 @@ prels AS ( THEN valtraduite ELSE 0 END - ) AS valtraduite_no3_no2 + ) AS valtraduite_no3_no2, + MAX( + CASE + WHEN + cdparametresiseeaux = 'NO3' + THEN limite_qualite + ELSE 0 + END + ) AS limite_qualite_no3, + MAX( + CASE + WHEN + cdparametresiseeaux = 'NO3_NO2' + THEN limite_qualite + ELSE 0 + END + ) AS limite_qualite_no3_no2, + MAX( + CASE + WHEN + cdparametresiseeaux = 'NO2' + THEN limite_qualite + ELSE 0 + END + ) AS limite_qualite_no2 FROM {{ ref('int__resultats_udi_communes') }} WHERE - categorie = 'nitrite' + categorie = 'nitrate' GROUP BY annee, cdreseau, @@ -45,15 +69,15 @@ prels AS ( SELECT prels.cdreseau, prels.annee, - 'nitrite' AS categorie, + 'nitrate' AS categorie, 'bilan_annuel_' || prels.annee AS periode, COUNT( DISTINCT CASE WHEN - prels.valtraduite_no3 >= 50 - OR prels.valtraduite_no2 >= 0.5 - OR prels.valtraduite_no3_no2 >= 1 + prels.valtraduite_no3 >= prels.limite_qualite_no3 + OR prels.valtraduite_no2 >= prels.limite_qualite_no2 + OR prels.valtraduite_no3_no2 >= prels.limite_qualite_no3_no2 THEN prels.referenceprel END ) AS nb_depassements, @@ -63,9 +87,9 @@ SELECT DISTINCT CASE WHEN - prels.valtraduite_no3 >= 50 - OR prels.valtraduite_no2 >= 0.5 - OR prels.valtraduite_no3_no2 >= 1 + prels.valtraduite_no3 >= prels.limite_qualite_no3 + OR prels.valtraduite_no2 >= prels.limite_qualite_no2 + OR prels.valtraduite_no3_no2 >= prels.limite_qualite_no3_no2 THEN prels.referenceprel END )::float diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index 14a71d99..3f6fc3c9 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -6,6 +6,7 @@ last_pvl AS ( cdparametresiseeaux, datetimeprel, valtraduite, + limite_qualite, ROW_NUMBER() OVER ( PARTITION BY cdreseau, cdparametresiseeaux @@ -15,7 +16,7 @@ last_pvl AS ( FROM {{ ref('int__resultats_udi_communes') }} WHERE - categorie = 'nitrite' + categorie = 'nitrate' AND -- On garde les prélèvements de moins d'un an CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR @@ -50,7 +51,31 @@ split_nitrites AS ( THEN last_pvl.valtraduite ELSE 0 END - ) AS valtraduite_no3_no2 + ) AS valtraduite_no3_no2, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3' + THEN last_pvl.limite_qualite + ELSE 0 + END + ) AS limite_qualite_no3, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3_NO2' + THEN last_pvl.limite_qualite + ELSE 0 + END + ) AS limite_qualite_no3_no2, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO2' + THEN last_pvl.limite_qualite + ELSE 0 + END + ) AS limite_qualite_no2 FROM last_pvl WHERE @@ -74,21 +99,29 @@ SELECT THEN 'non_quantifie' WHEN split_nitrites.nb_parametres = 3 - AND split_nitrites.valtraduite_no3 < 50 - AND split_nitrites.valtraduite_no2 < 0.5 - AND split_nitrites.valtraduite_no3_no2 < 1 + AND split_nitrites.valtraduite_no3 + < split_nitrites.limite_qualite_no3 + AND split_nitrites.valtraduite_no2 + < split_nitrites.limite_qualite_no2 + AND split_nitrites.valtraduite_no3_no2 + < split_nitrites.limite_qualite_no3_no2 THEN 'conforme' WHEN - split_nitrites.valtraduite_no3 >= 50 - OR split_nitrites.valtraduite_no2 >= 0.5 - OR split_nitrites.valtraduite_no3_no2 >= 1 + split_nitrites.valtraduite_no3 >= split_nitrites.limite_qualite_no3 + OR split_nitrites.valtraduite_no2 + >= split_nitrites.limite_qualite_no2 + OR split_nitrites.valtraduite_no3_no2 + >= split_nitrites.limite_qualite_no3_no2 THEN 'non_conforme' WHEN split_nitrites.nb_parametres != 3 AND ( - split_nitrites.valtraduite_no3 < 50 - OR split_nitrites.valtraduite_no2 < 0.5 - OR split_nitrites.valtraduite_no3_no2 < 1 + split_nitrites.valtraduite_no3 + < split_nitrites.limite_qualite_no3 + OR split_nitrites.valtraduite_no2 + < split_nitrites.limite_qualite_no2 + OR split_nitrites.valtraduite_no3_no2 + < split_nitrites.limite_qualite_no3_no2 ) THEN 'non_quantifie' ELSE 'error' From b312509a9a64cb82107b4861c80a3455bb286eeb Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 11 Apr 2025 21:17:49 +0200 Subject: [PATCH 10/30] feat: use new file GF + Update filed resultats --- analytics/notebooks/resultats_nitrates.ipynb | 823 +++++++++++------- .../nitrites/_int__nitrites_models.yaml | 2 +- .../int__resultats_nitrites_udi_dernier.sql | 128 +-- dbt_/tests/test_nitrates_results.sql | 26 +- 4 files changed, 595 insertions(+), 384 deletions(-) diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb index ac281159..c571db67 100644 --- a/analytics/notebooks/resultats_nitrates.ipynb +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -63,82 +63,185 @@ " \n", " \n", " cdparametresiseeaux\n", - " cdparametre\n", - " libmajparametre\n", - " libminparametre\n", - " casparam\n", - " categorie\n", + " categorie_1\n", + " categorie_2\n", + " categorie_3\n", + " limite_qualite\n", + " limite_qualite_unite\n", + " valeur_sanitaire_1\n", + " valeur_sanitaire_1_unite\n", + " valeur_sanitaire_2\n", + " valeur_sanitaire_2_unite\n", + " nb_rows\n", " \n", " \n", " \n", " \n", - " NO3\n", - " 1340\n", - " NITRATES (EN NO3)\n", - " Nitrates (en NO3)\n", + " NO2\n", + " nitrate\n", + " None\n", + " None\n", + " 0.5\n", + " mg/L\n", " None\n", - " nitrite\n", + " None\n", + " None\n", + " None\n", + " 1\n", " \n", " \n", - " NO3_NO2\n", - " 6374\n", - " NITRATES/50 + NITRITES/3\n", - " Nitrates/50 + Nitrites/3\n", + " NO3\n", + " nitrate\n", + " None\n", + " None\n", + " 50.0\n", + " mg/L\n", + " None\n", + " None\n", " None\n", - " nitrite\n", + " None\n", + " 1\n", " \n", " \n", - " NO2\n", - " 1339\n", - " NITRITES (EN NO2)\n", - " Nitrites (en NO2)\n", + " NO3_NO2\n", + " nitrate\n", + " None\n", " None\n", - " nitrite\n", + " 1.0\n", + " None\n", + " None\n", + " None\n", + " None\n", + " None\n", + " 1\n", " \n", + " \n", + "" + ], + "text/plain": [ + "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+\n", + "| cdparametresiseeaux | categorie_1 | categorie_2 | categorie_3 | limite_qualite | limite_qualite_unite | valeur_sanitaire_1 | valeur_sanitaire_1_unite | valeur_sanitaire_2 | valeur_sanitaire_2_unite | nb_rows |\n", + "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+\n", + "| NO2 | nitrate | None | None | 0.5 | mg/L | None | None | None | None | 1 |\n", + "| NO3 | nitrate | None | None | 50.0 | mg/L | None | None | None | None | 1 |\n", + "| NO3_NO2 | nitrate | None | None | 1.0 | None | None | None | None | None | 1 |\n", + "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "SELECT * FROM int__valeurs_de_reference WHERE categorie_1='nitrate'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1aff16f9-21a7-4e2b-a3f6-17fc453d7dbc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", "
NO31340NITRATES (EN NO3)Nitrates (en NO3)14797-55-8nitritelimite_qualite_no3limite_qualite_no3_no2limite_qualite_no2
NO21339NITRITES (EN NO2)Nitrites (en NO2)14797-65-0nitrite50.01.00.5
" ], "text/plain": [ - "+---------------------+-------------+--------------------------+--------------------------+------------+-----------+\n", - "| cdparametresiseeaux | cdparametre | libmajparametre | libminparametre | casparam | categorie |\n", - "+---------------------+-------------+--------------------------+--------------------------+------------+-----------+\n", - "| NO3 | 1340 | NITRATES (EN NO3) | Nitrates (en NO3) | None | nitrite |\n", - "| NO3_NO2 | 6374 | NITRATES/50 + NITRITES/3 | Nitrates/50 + Nitrites/3 | None | nitrite |\n", - "| NO2 | 1339 | NITRITES (EN NO2) | Nitrites (en NO2) | None | nitrite |\n", - "| NO3 | 1340 | NITRATES (EN NO3) | Nitrates (en NO3) | 14797-55-8 | nitrite |\n", - "| NO2 | 1339 | NITRITES (EN NO2) | Nitrites (en NO2) | 14797-65-0 | nitrite |\n", - "+---------------------+-------------+--------------------------+--------------------------+------------+-----------+" + "+--------------------+------------------------+--------------------+\n", + "| limite_qualite_no3 | limite_qualite_no3_no2 | limite_qualite_no2 |\n", + "+--------------------+------------------------+--------------------+\n", + "| 50.0 | 1.0 | 0.5 |\n", + "+--------------------+------------------------+--------------------+" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%sql\n", - "SELECT * FROM mapping_categories WHERE categorie='nitrite'" + "SELECT\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no2\n", + "FROM\n", + " int__valeurs_de_reference\n", + "WHERE\n", + " categorie_1 = 'nitrate'" + ] + }, + { + "cell_type": "markdown", + "id": "34654cff-fb90-4224-a5db-8086186a23f1", + "metadata": {}, + "source": [ + "**Les règles choisies**\n", + " \n", + "_Si on a les 3 paramètres disponibles_\n", + "- Si Nitrates < 50 mg/L et nitrites < 0,5 mg/L et nitrate/50 + nitrites/3 < 1 mg/L --> CONFORME\n", + "- Si Nitrates >= 50 mg/L et/ou nitrites >= 0,5 mg/L et/ou nitrate/50 + nitrites/3 >= 1 mg/L --> NON CONFORME\n", + "\n", + "\n", + "_Si on n'a pas nitrate/50 + nitrites/3 mais qu'on a nitrate et nitrites_
\n", + "==> on le calcule et on applique comme ci-dessus\n", + "\n", + "- Si on n'a pas nitrate --> donnee_manquante\n", + "- Si on n'a pas nitrites et que nitrate >= 50 mg/L --> NON CONFORME\n", + "- Si on n'a pas nitrites et que nitrate < 50 mg/L --> donnee_manquant \n" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "de799aa8-64a7-4423-901c-7de0fc237b70", "metadata": {}, "outputs": [ @@ -157,7 +260,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "393adae3495f4ce48ef2f5c79b4a58f1", + "model_id": "2abb38d323d04b04a1fcab8e91187bc0", "version_major": 2, "version_minor": 0 }, @@ -184,82 +287,82 @@ " \n", " \n", " \n", - " 038001345\n", - " nitrite\n", + " 038000406\n", + " nitrate\n", " dernier_prel\n", - " 2024-12-03 09:10:00\n", + " 2024-11-04 10:30:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001363\n", - " nitrite\n", + " 038000579\n", + " nitrate\n", " dernier_prel\n", - " 2024-10-17 10:15:00\n", + " 2024-10-08 10:45:00\n", " 3\n", - " donnee_manquante\n", + " conforme\n", " \n", " \n", - " 038001404\n", - " nitrite\n", + " 038000634\n", + " nitrate\n", " dernier_prel\n", - " 2024-09-13 08:05:00\n", + " 2025-01-30 09:20:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001576\n", - " nitrite\n", + " 038000869\n", + " nitrate\n", " dernier_prel\n", - " 2024-10-15 10:10:00\n", + " 2024-10-29 08:45:00\n", " 3\n", - " donnee_manquante\n", + " conforme\n", " \n", " \n", - " 038001625\n", - " nitrite\n", + " 038000919\n", + " nitrate\n", " dernier_prel\n", - " 2024-12-17 10:25:00\n", + " 2025-02-27 09:48:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001899\n", - " nitrite\n", + " 038000992\n", + " nitrate\n", " dernier_prel\n", - " 2024-11-13 07:35:00\n", + " 2025-01-02 10:35:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001960\n", - " nitrite\n", + " 038001014\n", + " nitrate\n", " dernier_prel\n", - " 2024-10-14 11:25:00\n", + " 2025-01-15 10:10:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038002035\n", - " nitrite\n", + " 038001205\n", + " nitrate\n", " dernier_prel\n", - " 2024-10-09 08:55:00\n", + " 2024-11-14 09:35:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038002151\n", - " nitrite\n", + " 038001237\n", + " nitrate\n", " dernier_prel\n", - " 2024-11-18 10:25:00\n", + " 2024-06-25 10:05:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038002176\n", - " nitrite\n", + " 038001241\n", + " nitrate\n", " dernier_prel\n", - " 2024-10-07 06:50:00\n", + " 2024-12-19 09:30:00\n", " 3\n", " conforme\n", " \n", @@ -268,24 +371,24 @@ "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 038001345 | nitrite | dernier_prel | 2024-12-03 09:10:00 | 3 | conforme |\n", - "| 038001363 | nitrite | dernier_prel | 2024-10-17 10:15:00 | 3 | donnee_manquante |\n", - "| 038001404 | nitrite | dernier_prel | 2024-09-13 08:05:00 | 3 | conforme |\n", - "| 038001576 | nitrite | dernier_prel | 2024-10-15 10:10:00 | 3 | donnee_manquante |\n", - "| 038001625 | nitrite | dernier_prel | 2024-12-17 10:25:00 | 3 | conforme |\n", - "| 038001899 | nitrite | dernier_prel | 2024-11-13 07:35:00 | 3 | conforme |\n", - "| 038001960 | nitrite | dernier_prel | 2024-10-14 11:25:00 | 3 | conforme |\n", - "| 038002035 | nitrite | dernier_prel | 2024-10-09 08:55:00 | 3 | conforme |\n", - "| 038002151 | nitrite | dernier_prel | 2024-11-18 10:25:00 | 3 | conforme |\n", - "| 038002176 | nitrite | dernier_prel | 2024-10-07 06:50:00 | 3 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", + "| 038000406 | nitrate | dernier_prel | 2024-11-04 10:30:00 | 3 | conforme |\n", + "| 038000579 | nitrate | dernier_prel | 2024-10-08 10:45:00 | 3 | conforme |\n", + "| 038000634 | nitrate | dernier_prel | 2025-01-30 09:20:00 | 3 | conforme |\n", + "| 038000869 | nitrate | dernier_prel | 2024-10-29 08:45:00 | 3 | conforme |\n", + "| 038000919 | nitrate | dernier_prel | 2025-02-27 09:48:00 | 3 | conforme |\n", + "| 038000992 | nitrate | dernier_prel | 2025-01-02 10:35:00 | 3 | conforme |\n", + "| 038001014 | nitrate | dernier_prel | 2025-01-15 10:10:00 | 3 | conforme |\n", + "| 038001205 | nitrate | dernier_prel | 2024-11-14 09:35:00 | 3 | conforme |\n", + "| 038001237 | nitrate | dernier_prel | 2024-06-25 10:05:00 | 3 | conforme |\n", + "| 038001241 | nitrate | dernier_prel | 2024-12-19 09:30:00 | 3 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", "Truncated to displaylimit of 10." ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -298,6 +401,7 @@ " cdreseau,\n", " categorie,\n", " cdparametresiseeaux,\n", + " limite_qualite,\n", " datetimeprel,\n", " valtraduite,\n", " ROW_NUMBER()\n", @@ -307,41 +411,49 @@ " )\n", " AS row_number\n", " FROM\n", - " int__resultats_udi_communes\n", + " int__resultats_udi_communes\n", " WHERE\n", - " categorie = 'nitrite'\n", + " categorie = 'nitrate'\n", " AND\n", " -- On garde les prélèvements de moins d'un an\n", " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", "),\n", "\n", - "split_nitrites AS (\n", + "valeur_ref AS (\n", " SELECT\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie,\n", - " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", - " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.datetimeprel\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", " END\n", - " ) AS dernier_prel_datetime_n03,\n", + " ) AS limite_qualite_no3,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.datetimeprel\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", " END\n", - " ) AS dernier_prel_datetime_n02,\n", + " ) AS limite_qualite_no3_no2,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", - " THEN last_pvl.datetimeprel\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", " END\n", - " ) AS dernier_prel_datetime_no3_no2,\n", + " ) AS limite_qualite_no2\n", + " FROM\n", + " int__valeurs_de_reference\n", + " WHERE\n", + " categorie_1 = 'nitrate'\n", + "),\n", + "\n", + "split_nitrites AS (\n", + " SELECT\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie,\n", + " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", + " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", " MAX(\n", " CASE\n", " WHEN\n", @@ -370,70 +482,149 @@ " GROUP BY\n", " last_pvl.cdreseau,\n", " last_pvl.categorie\n", + "),\n", + "\n", + "split_nitrites_with_ref AS (\n", + " SELECT\n", + " split_nitrites.*,\n", + " valeur_ref.limite_qualite_no3,\n", + " valeur_ref.limite_qualite_no3_no2,\n", + " valeur_ref.limite_qualite_no2,\n", + " split_nitrites.valtraduite_no3 / 50\n", + " + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc\n", + " FROM split_nitrites\n", + " CROSS JOIN valeur_ref\n", + "\n", ")\n", "\n", "SELECT\n", - " split_nitrites.cdreseau,\n", - " split_nitrites.categorie,\n", + " split_nitrites_with_ref.cdreseau,\n", + " split_nitrites_with_ref.categorie,\n", " 'dernier_prel' AS periode,\n", - " split_nitrites.dernier_prel_datetime,\n", - " split_nitrites.nb_parametres,\n", + " split_nitrites_with_ref.dernier_prel_datetime,\n", + " split_nitrites_with_ref.nb_parametres,\n", " CASE\n", " WHEN\n", - " DATE_DIFF(\n", - " 'day',\n", - " LEAST(\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02,\n", - " split_nitrites.dernier_prel_datetime_no3_no2\n", - " ),\n", - " GREATEST(\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02,\n", - " split_nitrites.dernier_prel_datetime_no3_no2\n", - " )\n", - " ) >= 30\n", - " OR split_nitrites.valtraduite_no2 IS NULL\n", - " OR split_nitrites.valtraduite_no3 IS NULL\n", - " THEN 'donnee_manquante'\n", - " WHEN\n", - " split_nitrites.valtraduite_no3_no2 IS NULL\n", - " AND split_nitrites.valtraduite_no2 IS NOT NULL\n", - " AND split_nitrites.valtraduite_no3 IS NOT NULL\n", - " AND split_nitrites.valtraduite_no3 < 50\n", - " AND split_nitrites.valtraduite_no2 < 0.5\n", - " AND split_nitrites.valtraduite_no3 / 50\n", - " + split_nitrites.valtraduite_no2 / 3\n", - " < 1\n", - " THEN 'conforme'\n", - " WHEN\n", - " split_nitrites.nb_parametres = 3\n", - " AND split_nitrites.valtraduite_no3 < 50\n", - " AND split_nitrites.valtraduite_no2 < 0.5\n", - " AND split_nitrites.valtraduite_no3_no2 < 1\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", + " AND split_nitrites_with_ref.valtraduite_no2\n", + " < split_nitrites_with_ref.limite_qualite_no2\n", + " AND COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " split_nitrites_with_ref.valtraduite_no3_no2_calc\n", + " )\n", + " < split_nitrites_with_ref.limite_qualite_no3_no2\n", " THEN 'conforme'\n", " WHEN\n", - " split_nitrites.valtraduite_no3 >= 50\n", - " OR split_nitrites.valtraduite_no2 >= 0.5\n", - " OR split_nitrites.valtraduite_no3_no2 >= 1\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " >= split_nitrites_with_ref.limite_qualite_no3\n", + " OR split_nitrites_with_ref.valtraduite_no2\n", + " >= split_nitrites_with_ref.limite_qualite_no2\n", + " OR COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " split_nitrites_with_ref.valtraduite_no3_no2_calc\n", + " )\n", + " >= split_nitrites_with_ref.limite_qualite_no3_no2\n", " THEN 'non_conforme'\n", - " /* WHEN\n", - " split_nitrites.nb_parametres != 3\n", - " AND (\n", - " split_nitrites.valtraduite_no3 < 50\n", - " OR split_nitrites.valtraduite_no2 < 0.5\n", - " OR split_nitrites.valtraduite_no3_no2 < 1\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3 IS NULL\n", + " OR (\n", + " split_nitrites_with_ref.valtraduite_no2 IS NULL\n", + " AND split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", " )\n", - " THEN 'non_quantifie'*/\n", + " THEN 'donnee_manquante'\n", " ELSE 'error'\n", " END AS resultat\n", "FROM\n", - " split_nitrites\n" + " split_nitrites_with_ref\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, + "id": "fe5821d3-419f-46af-92d4-311f50d4a9b7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7cef2c5b68ab42ad94b21ca3d9e681ac", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
resultatcount_star()
non_conforme269
conforme20741
donnee_manquante1149
" + ], + "text/plain": [ + "+------------------+--------------+\n", + "| resultat | count_star() |\n", + "+------------------+--------------+\n", + "| non_conforme | 269 |\n", + "| conforme | 20741 |\n", + "| donnee_manquante | 1149 |\n", + "+------------------+--------------+" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier\n", + "SELECT \n", + " resultat, COUNT(*)\n", + "FROM \n", + " int__resultats_nitrites_udi_dernier\n", + "GROUP BY 1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, "id": "2a287b57-fa15-4939-8209-7dcdf0e75eda", "metadata": {}, "outputs": [ @@ -452,7 +643,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "929b4592e19847a2802cb22edcf32040", + "model_id": "cee2a9510a9748648016d352dee74573", "version_major": 2, "version_minor": 0 }, @@ -488,7 +679,7 @@ "+----------+-----------+---------+-----------------------+---------------+----------+" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -505,7 +696,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "id": "30487685-f619-43ab-be20-054cb46db3c6", "metadata": {}, "outputs": [ @@ -524,7 +715,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4c142c15520f4e5a9e4d583196cc29e4", + "model_id": "53e139d8f8af457b8b9510adacb72410", "version_major": 2, "version_minor": 0 }, @@ -548,19 +739,19 @@ " \n", " \n", " \n", - " non_conforme\n", - " 002000060\n", - " 095000307\n", - " \n", - " \n", " conforme\n", " 001000003\n", - " 976003487\n", + " 976003489\n", " \n", " \n", " donnee_manquante\n", " 001000251\n", - " 976003554\n", + " 976003881\n", + " \n", + " \n", + " non_conforme\n", + " 002000060\n", + " 095000346\n", " \n", " \n", "" @@ -569,13 +760,13 @@ "+------------------+--------------+--------------+\n", "| resultat | min_cdreseau | max_cdreseau |\n", "+------------------+--------------+--------------+\n", - "| non_conforme | 002000060 | 095000307 |\n", - "| conforme | 001000003 | 976003487 |\n", - "| donnee_manquante | 001000251 | 976003554 |\n", + "| conforme | 001000003 | 976003489 |\n", + "| donnee_manquante | 001000251 | 976003881 |\n", + "| non_conforme | 002000060 | 095000346 |\n", "+------------------+--------------+--------------+" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -618,7 +809,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "08da46f805e6474bb69cb35f30140197", + "model_id": "d2e860daff024a4491bcec409b322be2", "version_major": 2, "version_minor": 0 }, @@ -637,157 +828,132 @@ " \n", " cdreseau\n", " categorie\n", - " nb_parametres\n", + " periode\n", " dernier_prel_datetime\n", - " dernier_prel_datetime_n03\n", - " dernier_prel_datetime_n02\n", - " dernier_prel_datetime_no3_no2\n", - " valtraduite_no3\n", + " nb_parametres\n", " valtraduite_no2\n", + " valtraduite_no3\n", " valtraduite_no3_no2\n", + " valtraduite_no3_no2_calc\n", " \n", " \n", " \n", " \n", " 001000003\n", - " nitrite\n", + " nitrate\n", + " dernier_prel\n", + " 2025-01-21 12:35:00\n", " 3\n", - " 2024-12-26 09:08:00\n", - " 2024-12-26 09:08:00\n", - " 2024-12-26 09:08:00\n", - " 2024-12-26 09:08:00\n", - " 13.000\n", " 0.000\n", - " 0.260\n", + " 14.000\n", + " 0.280\n", + " 0.28\n", " \n", " \n", " 001000260\n", - " nitrite\n", - " 3\n", - " 2024-12-06 12:02:00\n", - " 2024-12-06 12:02:00\n", - " 2024-12-06 12:02:00\n", + " nitrate\n", + " dernier_prel\n", " 2024-12-06 12:02:00\n", - " 2.400\n", + " 3\n", " 0.000\n", + " 2.400\n", " 0.050\n", + " 0.048\n", " \n", " \n", " 001000268\n", - " nitrite\n", + " nitrate\n", + " dernier_prel\n", + " 2025-01-13 09:57:00\n", " 3\n", - " 2024-11-28 08:25:00\n", - " 2024-11-28 08:25:00\n", - " 2024-11-28 08:25:00\n", - " 2024-11-28 08:25:00\n", - " 2.600\n", " 0.000\n", - " 0.050\n", + " 6.000\n", + " 0.120\n", + " 0.12\n", " \n", " \n", " 001000293\n", - " nitrite\n", + " nitrate\n", + " dernier_prel\n", + " 2025-01-28 11:29:00\n", " 3\n", - " 2024-10-10 10:20:00\n", - " 2024-10-10 10:20:00\n", - " 2024-10-10 10:20:00\n", - " 2024-10-10 10:20:00\n", - " 4.000\n", " 0.000\n", - " 0.080\n", + " 3.700\n", + " 0.070\n", + " 0.07400000000000001\n", " \n", " \n", " 002000060\n", - " nitrite\n", - " 3\n", - " 2024-10-18 08:50:00\n", - " 2024-10-18 08:50:00\n", + " nitrate\n", + " dernier_prel\n", " 2024-10-18 08:50:00\n", - " 2024-10-18 08:31:00\n", - " 59.300\n", + " 3\n", " 0.000\n", + " 59.300\n", " 0.000\n", + " 1.186\n", " \n", " \n", " 073002059\n", - " nitrite\n", - " 1\n", - " 2024-10-23 09:28:00\n", - " None\n", + " nitrate\n", + " dernier_prel\n", " 2024-10-23 09:28:00\n", + " 1\n", + " 0.000\n", " None\n", " None\n", - " 0.000\n", " None\n", " \n", " \n", " 088002296\n", - " nitrite\n", - " 1\n", - " 2024-09-27 12:00:00\n", - " None\n", + " nitrate\n", + " dernier_prel\n", " 2024-09-27 12:00:00\n", + " 1\n", + " 0.000\n", " None\n", " None\n", - " 0.000\n", " None\n", " \n", " \n", " 095000346\n", - " nitrite\n", + " nitrate\n", + " dernier_prel\n", + " 2025-02-27 08:35:00\n", " 3\n", - " 2024-12-17 08:55:00\n", - " 2024-12-17 08:55:00\n", - " 2024-07-01 09:20:00\n", - " 2024-07-01 09:20:00\n", - " 51.000\n", " 0.000\n", + " 47.000\n", " 1.000\n", + " 0.94\n", " \n", " \n", " 973000028\n", - " nitrite\n", - " 2\n", - " 2024-12-02 11:30:00\n", - " 2024-12-02 11:30:00\n", + " nitrate\n", + " dernier_prel\n", " 2024-12-02 11:30:00\n", - " None\n", - " 0.000\n", - " 0.000\n", - " None\n", - " \n", - " \n", - " 976003554\n", - " nitrite\n", " 2\n", - " 2024-11-12 10:00:00\n", - " 2024-11-12 10:00:00\n", - " 2024-09-10 09:20:00\n", - " None\n", " 0.000\n", " 0.000\n", " None\n", + " 0.0\n", " \n", " \n", - "\n", - "Truncated to displaylimit of 10." + "" ], "text/plain": [ - "+-----------+-----------+---------------+-----------------------+---------------------------+---------------------------+-------------------------------+-----------------+-----------------+---------------------+\n", - "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | dernier_prel_datetime_n03 | dernier_prel_datetime_n02 | dernier_prel_datetime_no3_no2 | valtraduite_no3 | valtraduite_no2 | valtraduite_no3_no2 |\n", - "+-----------+-----------+---------------+-----------------------+---------------------------+---------------------------+-------------------------------+-----------------+-----------------+---------------------+\n", - "| 001000003 | nitrite | 3 | 2024-12-26 09:08:00 | 2024-12-26 09:08:00 | 2024-12-26 09:08:00 | 2024-12-26 09:08:00 | 13.000 | 0.000 | 0.260 |\n", - "| 001000260 | nitrite | 3 | 2024-12-06 12:02:00 | 2024-12-06 12:02:00 | 2024-12-06 12:02:00 | 2024-12-06 12:02:00 | 2.400 | 0.000 | 0.050 |\n", - "| 001000268 | nitrite | 3 | 2024-11-28 08:25:00 | 2024-11-28 08:25:00 | 2024-11-28 08:25:00 | 2024-11-28 08:25:00 | 2.600 | 0.000 | 0.050 |\n", - "| 001000293 | nitrite | 3 | 2024-10-10 10:20:00 | 2024-10-10 10:20:00 | 2024-10-10 10:20:00 | 2024-10-10 10:20:00 | 4.000 | 0.000 | 0.080 |\n", - "| 002000060 | nitrite | 3 | 2024-10-18 08:50:00 | 2024-10-18 08:50:00 | 2024-10-18 08:50:00 | 2024-10-18 08:31:00 | 59.300 | 0.000 | 0.000 |\n", - "| 073002059 | nitrite | 1 | 2024-10-23 09:28:00 | None | 2024-10-23 09:28:00 | None | None | 0.000 | None |\n", - "| 088002296 | nitrite | 1 | 2024-09-27 12:00:00 | None | 2024-09-27 12:00:00 | None | None | 0.000 | None |\n", - "| 095000346 | nitrite | 3 | 2024-12-17 08:55:00 | 2024-12-17 08:55:00 | 2024-07-01 09:20:00 | 2024-07-01 09:20:00 | 51.000 | 0.000 | 1.000 |\n", - "| 973000028 | nitrite | 2 | 2024-12-02 11:30:00 | 2024-12-02 11:30:00 | 2024-12-02 11:30:00 | None | 0.000 | 0.000 | None |\n", - "| 976003554 | nitrite | 2 | 2024-11-12 10:00:00 | 2024-11-12 10:00:00 | 2024-09-10 09:20:00 | None | 0.000 | 0.000 | None |\n", - "+-----------+-----------+---------------+-----------------------+---------------------------+---------------------------+-------------------------------+-----------------+-----------------+---------------------+\n", - "Truncated to displaylimit of 10." + "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | valtraduite_no2 | valtraduite_no3 | valtraduite_no3_no2 | valtraduite_no3_no2_calc |\n", + "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+\n", + "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | 0.000 | 14.000 | 0.280 | 0.28 |\n", + "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | 0.000 | 2.400 | 0.050 | 0.048 |\n", + "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | 0.000 | 6.000 | 0.120 | 0.12 |\n", + "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | 0.000 | 3.700 | 0.070 | 0.07400000000000001 |\n", + "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | 0.000 | 59.300 | 0.000 | 1.186 |\n", + "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | 0.000 | None | None | None |\n", + "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | 0.000 | None | None | None |\n", + "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | 0.000 | 47.000 | 1.000 | 0.94 |\n", + "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | 0.000 | 0.000 | None | 0.0 |\n", + "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+" ] }, "execution_count": 8, @@ -803,6 +969,7 @@ " cdreseau,\n", " categorie,\n", " cdparametresiseeaux,\n", + " limite_qualite,\n", " datetimeprel,\n", " valtraduite,\n", " ROW_NUMBER()\n", @@ -812,13 +979,42 @@ " )\n", " AS row_number\n", " FROM\n", - " int__resultats_udi_communes\n", + " int__resultats_udi_communes\n", " WHERE\n", - " categorie = 'nitrite'\n", + " categorie = 'nitrate'\n", " AND\n", " -- On garde les prélèvements de moins d'un an\n", " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", - ")\n", + "),\n", + "\n", + "valeur_ref AS (\n", + "SELECT \n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no2\n", + "FROM int__valeurs_de_reference\n", + "WHERE categorie_1='nitrate' \n", + "),\n", + "\n", + "split_nitrites AS (\n", " SELECT\n", " last_pvl.cdreseau,\n", " last_pvl.categorie,\n", @@ -870,17 +1066,45 @@ " last_pvl\n", " WHERE\n", " last_pvl.row_number = 1\n", - " AND cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", " GROUP BY\n", " last_pvl.cdreseau,\n", " last_pvl.categorie\n", - " ORDER BY \n", - " cdreseau" + "),\n", + "\n", + "split_nitrites_with_ref AS (\n", + " SELECT \n", + " split_nitrites.*,\n", + " valeur_ref.limite_qualite_no3,\n", + " valeur_ref.limite_qualite_no3_no2,\n", + " valeur_ref.limite_qualite_no2,\n", + " split_nitrites.valtraduite_no3 / 50\n", + " + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc\n", + " FROM split_nitrites\n", + " CROSS JOIN valeur_ref\n", + "\n", + ")\n", + "\n", + "SELECT\n", + " split_nitrites_with_ref.cdreseau,\n", + " split_nitrites_with_ref.categorie,\n", + " 'dernier_prel' AS periode,\n", + " split_nitrites_with_ref.dernier_prel_datetime,\n", + " split_nitrites_with_ref.nb_parametres,\n", + " split_nitrites_with_ref.valtraduite_no2,\n", + " split_nitrites_with_ref.valtraduite_no3,\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " split_nitrites_with_ref.valtraduite_no3_no2_calc\n", + "FROM\n", + " split_nitrites_with_ref \n", + "WHERE\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", + "ORDER BY \n", + " cdreseau" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "2137dd22-ec22-4b37-93ee-30b75c06ad42", "metadata": {}, "outputs": [ @@ -899,7 +1123,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f39eb246f635494e933f46977dd4b3d9", + "model_id": "9249792e8cbd438ab76d46ddb8001245", "version_major": 2, "version_minor": 0 }, @@ -927,15 +1151,15 @@ " \n", " \n", " 001000003\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", - " 2024-12-26 09:08:00\n", + " 2025-01-21 12:35:00\n", " 3\n", " conforme\n", " \n", " \n", " 001000260\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", " 2024-12-06 12:02:00\n", " 3\n", @@ -943,23 +1167,23 @@ " \n", " \n", " 001000268\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", - " 2024-11-28 08:25:00\n", + " 2025-01-13 09:57:00\n", " 3\n", " conforme\n", " \n", " \n", " 001000293\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", - " 2024-10-10 10:20:00\n", + " 2025-01-28 11:29:00\n", " 3\n", " conforme\n", " \n", " \n", " 002000060\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", " 2024-10-18 08:50:00\n", " 3\n", @@ -967,7 +1191,7 @@ " \n", " \n", " 073002059\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", " 2024-10-23 09:28:00\n", " 1\n", @@ -975,7 +1199,7 @@ " \n", " \n", " 088002296\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", " 2024-09-27 12:00:00\n", " 1\n", @@ -983,51 +1207,40 @@ " \n", " \n", " 095000346\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", - " 2024-12-17 08:55:00\n", + " 2025-02-27 08:35:00\n", " 3\n", - " donnee_manquante\n", + " non_conforme\n", " \n", " \n", " 973000028\n", - " nitrite\n", + " nitrate\n", " dernier_prel\n", " 2024-12-02 11:30:00\n", " 2\n", " conforme\n", " \n", - " \n", - " 976003554\n", - " nitrite\n", - " dernier_prel\n", - " 2024-11-12 10:00:00\n", - " 2\n", - " donnee_manquante\n", - " \n", " \n", - "\n", - "Truncated to displaylimit of 10." + "" ], "text/plain": [ "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000003 | nitrite | dernier_prel | 2024-12-26 09:08:00 | 3 | conforme |\n", - "| 001000260 | nitrite | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", - "| 001000268 | nitrite | dernier_prel | 2024-11-28 08:25:00 | 3 | conforme |\n", - "| 001000293 | nitrite | dernier_prel | 2024-10-10 10:20:00 | 3 | conforme |\n", - "| 002000060 | nitrite | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", - "| 073002059 | nitrite | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", - "| 088002296 | nitrite | dernier_prel | 2024-09-27 12:00:00 | 1 | donnee_manquante |\n", - "| 095000346 | nitrite | dernier_prel | 2024-12-17 08:55:00 | 3 | donnee_manquante |\n", - "| 973000028 | nitrite | dernier_prel | 2024-12-02 11:30:00 | 2 | conforme |\n", - "| 976003554 | nitrite | dernier_prel | 2024-11-12 10:00:00 | 2 | donnee_manquante |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "Truncated to displaylimit of 10." + "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | conforme |\n", + "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", + "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | conforme |\n", + "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | conforme |\n", + "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", + "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", + "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | donnee_manquante |\n", + "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | non_conforme |\n", + "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -1039,7 +1252,7 @@ " FROM\n", " int__resultats_nitrites_udi_dernier\n", "WHERE\n", - " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293',)\n", "ORDER BY \n", " cdreseau" ] @@ -1047,7 +1260,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a7ee43cb-a514-4d48-8e4d-8fa6d95fdec4", + "id": "6dfd9bc4-176f-48ae-9db6-9b8be7b82b8d", "metadata": {}, "outputs": [], "source": [] diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index c0ea60e9..bfe74325 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -12,7 +12,7 @@ models: tests: - not_null - accepted_values: - values: ["nitrite"] + values: ["nitrate"] - name: periode type: VARCHAR tests: diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index 5c8ebb90..b1f85552 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -4,6 +4,7 @@ last_pvl AS ( cdreseau, categorie, cdparametresiseeaux, + limite_qualite, datetimeprel, valtraduite, ROW_NUMBER() @@ -15,39 +16,47 @@ last_pvl AS ( FROM {{ ref('int__resultats_udi_communes') }} WHERE - categorie = 'nitrite' + categorie = 'nitrate' AND -- On garde les prélèvements de moins d'un an CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR ), -split_nitrites AS ( +valeur_ref AS ( SELECT - last_pvl.cdreseau, - last_pvl.categorie, - COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres, - MAX(last_pvl.datetimeprel) AS dernier_prel_datetime, MAX( CASE WHEN - last_pvl.cdparametresiseeaux = 'NO3' - THEN last_pvl.datetimeprel + cdparametresiseeaux = 'NO3' + THEN limite_qualite END - ) AS dernier_prel_datetime_n03, + ) AS limite_qualite_no3, MAX( CASE WHEN - last_pvl.cdparametresiseeaux = 'NO2' - THEN last_pvl.datetimeprel + cdparametresiseeaux = 'NO3_NO2' + THEN limite_qualite END - ) AS dernier_prel_datetime_n02, + ) AS limite_qualite_no3_no2, MAX( CASE WHEN - last_pvl.cdparametresiseeaux = 'NO3_NO2' - THEN last_pvl.datetimeprel + cdparametresiseeaux = 'NO2' + THEN limite_qualite END - ) AS dernier_prel_datetime_no3_no2, + ) AS limite_qualite_no2 + FROM + int__valeurs_de_reference + WHERE + categorie_1 = 'nitrate' +), + +split_nitrites AS ( + SELECT + last_pvl.cdreseau, + last_pvl.categorie, + COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres, + MAX(last_pvl.datetimeprel) AS dernier_prel_datetime, MAX( CASE WHEN @@ -76,62 +85,59 @@ split_nitrites AS ( GROUP BY last_pvl.cdreseau, last_pvl.categorie +), + +split_nitrites_with_ref AS ( + SELECT + split_nitrites.*, + valeur_ref.limite_qualite_no3, + valeur_ref.limite_qualite_no3_no2, + valeur_ref.limite_qualite_no2, + split_nitrites.valtraduite_no3 / 50 + + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc + FROM split_nitrites + CROSS JOIN valeur_ref + ) SELECT - split_nitrites.cdreseau, - split_nitrites.categorie, + split_nitrites_with_ref.cdreseau, + split_nitrites_with_ref.categorie, 'dernier_prel' AS periode, - split_nitrites.dernier_prel_datetime, - split_nitrites.nb_parametres, + split_nitrites_with_ref.dernier_prel_datetime, + split_nitrites_with_ref.nb_parametres, CASE WHEN - DATE_DIFF( - 'day', - LEAST( - split_nitrites.dernier_prel_datetime_n03, - split_nitrites.dernier_prel_datetime_n02, - split_nitrites.dernier_prel_datetime_no3_no2 - ), - GREATEST( - split_nitrites.dernier_prel_datetime_n03, - split_nitrites.dernier_prel_datetime_n02, - split_nitrites.dernier_prel_datetime_no3_no2 - ) - ) >= 30 - OR split_nitrites.valtraduite_no2 IS NULL - OR split_nitrites.valtraduite_no3 IS NULL - THEN 'donnee_manquante' - WHEN - split_nitrites.valtraduite_no3_no2 IS NULL - AND split_nitrites.valtraduite_no2 IS NOT NULL - AND split_nitrites.valtraduite_no3 IS NOT NULL - AND split_nitrites.valtraduite_no3 < 50 - AND split_nitrites.valtraduite_no2 < 0.5 - AND split_nitrites.valtraduite_no3 / 50 - + split_nitrites.valtraduite_no2 / 3 - < 1 - THEN 'conforme' - WHEN - split_nitrites.nb_parametres = 3 - AND split_nitrites.valtraduite_no3 < 50 - AND split_nitrites.valtraduite_no2 < 0.5 - AND split_nitrites.valtraduite_no3_no2 < 1 + split_nitrites_with_ref.valtraduite_no3 + < split_nitrites_with_ref.limite_qualite_no3 + AND split_nitrites_with_ref.valtraduite_no2 + < split_nitrites_with_ref.limite_qualite_no2 + AND COALESCE( + split_nitrites_with_ref.valtraduite_no3_no2, + split_nitrites_with_ref.valtraduite_no3_no2_calc + ) + < split_nitrites_with_ref.limite_qualite_no3_no2 THEN 'conforme' WHEN - split_nitrites.valtraduite_no3 >= 50 - OR split_nitrites.valtraduite_no2 >= 0.5 - OR split_nitrites.valtraduite_no3_no2 >= 1 + split_nitrites_with_ref.valtraduite_no3 + >= split_nitrites_with_ref.limite_qualite_no3 + OR split_nitrites_with_ref.valtraduite_no2 + >= split_nitrites_with_ref.limite_qualite_no2 + OR COALESCE( + split_nitrites_with_ref.valtraduite_no3_no2, + split_nitrites_with_ref.valtraduite_no3_no2_calc + ) + >= split_nitrites_with_ref.limite_qualite_no3_no2 THEN 'non_conforme' - /* WHEN - split_nitrites.nb_parametres != 3 - AND ( - split_nitrites.valtraduite_no3 < 50 - OR split_nitrites.valtraduite_no2 < 0.5 - OR split_nitrites.valtraduite_no3_no2 < 1 + WHEN + split_nitrites_with_ref.valtraduite_no3 IS NULL + OR ( + split_nitrites_with_ref.valtraduite_no2 IS NULL + AND split_nitrites_with_ref.valtraduite_no3 + < split_nitrites_with_ref.limite_qualite_no3 ) - THEN 'non_quantifie'*/ + THEN 'donnee_manquante' ELSE 'error' END AS resultat FROM - split_nitrites + split_nitrites_with_ref diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index f9d2156c..0eb2157a 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -12,35 +12,35 @@ FROM WHERE ( cdreseau = '001000003' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-12-26 09:08:00' AND resultat != 'conforme' ) OR ( cdreseau = '001000260' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-12-06 12:02:00' AND resultat != 'conforme' ) OR ( cdreseau = '001000268' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-11-28 08:25:00' AND resultat != 'conforme' ) OR ( cdreseau = '002000060' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-10-18 08:50:00' AND resultat != 'non_conforme' ) OR ( cdreseau = '073002059' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-10-23 09:28:00' AND resultat != 'donnee_manquante' AND nb_parametres = 1 @@ -48,7 +48,7 @@ WHERE OR ( cdreseau = '088002296' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-09-27 12:00:00' AND resultat != 'donnee_manquante' AND nb_parametres = 1 @@ -56,23 +56,15 @@ WHERE OR ( cdreseau = '095000346' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-12-17 08:55:00' - AND resultat != 'donnee_manquante' + AND resultat != 'non_conforme' ) OR ( cdreseau = '973000028' - AND categorie = 'nitrite' + AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-12-17 08:55:00' AND resultat != 'conforme' AND nb_parametres = 2 ) - OR - ( - cdreseau = '976003554' - AND categorie = 'nitrite' - AND dernier_prel_datetime = '2024-11-12 10:00:00' - AND resultat != 'donnee_manquante' - AND nb_parametres = 2 - ) From 4ce549af748621bf98fe6bbe8098273e911da8b4 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 11 Apr 2025 21:49:17 +0200 Subject: [PATCH 11/30] wip --- .../int__resultats_nitrites_udi_annuel.sql | 50 +++++++++++-------- .../int__resultats_nitrites_udi_dernier.sql | 3 +- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql index fc0d6c90..7972d4a2 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql @@ -29,13 +29,25 @@ prels AS ( THEN valtraduite ELSE 0 END - ) AS valtraduite_no3_no2, + ) AS valtraduite_no3_no2 + FROM + {{ ref('int__resultats_udi_communes') }} + WHERE + categorie = 'nitrate' + GROUP BY + annee, + cdreseau, + referenceprel, + datetimeprel, +), + +valeur_ref AS ( + SELECT MAX( CASE WHEN cdparametresiseeaux = 'NO3' THEN limite_qualite - ELSE 0 END ) AS limite_qualite_no3, MAX( @@ -43,7 +55,6 @@ prels AS ( WHEN cdparametresiseeaux = 'NO3_NO2' THEN limite_qualite - ELSE 0 END ) AS limite_qualite_no3_no2, MAX( @@ -51,20 +62,13 @@ prels AS ( WHEN cdparametresiseeaux = 'NO2' THEN limite_qualite - ELSE 0 END ) AS limite_qualite_no2 FROM - {{ ref('int__resultats_udi_communes') }} + {{ ref('int__valeurs_de_reference') }} WHERE - categorie = 'nitrate' - GROUP BY - annee, - cdreseau, - referenceprel, - datetimeprel, -) - + categorie_1 = 'nitrate' +), SELECT prels.cdreseau, @@ -75,9 +79,12 @@ SELECT DISTINCT CASE WHEN - prels.valtraduite_no3 >= prels.limite_qualite_no3 - OR prels.valtraduite_no2 >= prels.limite_qualite_no2 - OR prels.valtraduite_no3_no2 >= prels.limite_qualite_no3_no2 + prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3 + OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2 + OR prels.valtraduite_no3_no2 + >= valeur_ref.limite_qualite_no3_no2 + OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3 + >= valeur_ref.limite_qualite_no3_no2 THEN prels.referenceprel END ) AS nb_depassements, @@ -87,9 +94,12 @@ SELECT DISTINCT CASE WHEN - prels.valtraduite_no3 >= prels.limite_qualite_no3 - OR prels.valtraduite_no2 >= prels.limite_qualite_no2 - OR prels.valtraduite_no3_no2 >= prels.limite_qualite_no3_no2 + prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3 + OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2 + OR prels.valtraduite_no3_no2 + >= valeur_ref.limite_qualite_no3_no2 + OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3 + >= valeur_ref.limite_qualite_no3_no2 THEN prels.referenceprel END )::float @@ -98,5 +108,5 @@ SELECT ) AS ratio_depassements FROM prels - +CROSS JOIN valeur_ref GROUP BY prels.cdreseau, prels.annee diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index 818c4f36..1fd644e3 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -7,7 +7,6 @@ last_pvl AS ( limite_qualite, datetimeprel, valtraduite, - limite_qualite, ROW_NUMBER() OVER ( PARTITION BY cdreseau, cdparametresiseeaux @@ -47,7 +46,7 @@ valeur_ref AS ( END ) AS limite_qualite_no2 FROM - int__valeurs_de_reference + {{ ref('int__valeurs_de_reference') }} WHERE categorie_1 = 'nitrate' ), From e5fb12523d3ff66f14cbd68ae08effdfce1f0924 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 11 Apr 2025 21:57:23 +0200 Subject: [PATCH 12/30] feat: add int__valeurs_de_reference and update nb_depassements --- .../nitrites/int__resultats_nitrites_udi_annuel.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql index 7972d4a2..71c883ca 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql @@ -68,7 +68,7 @@ valeur_ref AS ( {{ ref('int__valeurs_de_reference') }} WHERE categorie_1 = 'nitrate' -), +) SELECT prels.cdreseau, From 74ce684ae063883a595ac7d0758fc95c50a70e1d Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 11 Apr 2025 22:12:29 +0200 Subject: [PATCH 13/30] feat: add annuel test --- .../notebooks/resultats_nitrates_annuel.ipynb | 576 ++++++++++++++++++ dbt_/tests/test_nitrates_results.sql | 59 ++ 2 files changed, 635 insertions(+) create mode 100644 analytics/notebooks/resultats_nitrates_annuel.ipynb diff --git a/analytics/notebooks/resultats_nitrates_annuel.ipynb b/analytics/notebooks/resultats_nitrates_annuel.ipynb new file mode 100644 index 00000000..1acff7fb --- /dev/null +++ b/analytics/notebooks/resultats_nitrates_annuel.ipynb @@ -0,0 +1,576 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2e8d5bd2-d7c1-4ecb-805d-27eaa75195e0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "The 'toml' package isn't installed. To load settings from pyproject.toml or ~/.jupysql/config, install with: pip install toml" + ], + "text/plain": [ + "The 'toml' package isn't installed. To load settings from pyproject.toml or ~/.jupysql/config, install with: pip install toml" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Connecting to 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Connecting to 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%load_ext sql\n", + "%sql duckdb:///../../database/data.duckdb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3e1967e8-eba0-4261-bf9c-d16dd4a9b271", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdparametresiseeauxcategorie_1categorie_2categorie_3limite_qualitelimite_qualite_unitevaleur_sanitaire_1valeur_sanitaire_1_unitevaleur_sanitaire_2valeur_sanitaire_2_unitenb_rows
NO3nitrateNoneNone50.0mg/LNoneNoneNoneNone1
NO2nitrateNoneNone0.5mg/LNoneNoneNoneNone1
NO3_NO2nitrateNoneNone1.0NoneNoneNoneNoneNone1
" + ], + "text/plain": [ + "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+\n", + "| cdparametresiseeaux | categorie_1 | categorie_2 | categorie_3 | limite_qualite | limite_qualite_unite | valeur_sanitaire_1 | valeur_sanitaire_1_unite | valeur_sanitaire_2 | valeur_sanitaire_2_unite | nb_rows |\n", + "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+\n", + "| NO3 | nitrate | None | None | 50.0 | mg/L | None | None | None | None | 1 |\n", + "| NO2 | nitrate | None | None | 0.5 | mg/L | None | None | None | None | 1 |\n", + "| NO3_NO2 | nitrate | None | None | 1.0 | None | None | None | None | None | 1 |\n", + "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "SELECT * FROM int__valeurs_de_reference WHERE categorie_1='nitrate'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "de799aa8-64a7-4423-901c-7de0fc237b70", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3989ea173ae84381b3cddcb94c423fce", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseauanneecategorieperiodenb_depassementsnb_prelevementsratio_depassements
0100001442024nitratebilan_annuel_2024090.0
0320002022023nitratebilan_annuel_20230160.0
0880015652020nitratebilan_annuel_2020040.0
0210007872023nitratebilan_annuel_2023030.0
0170003742022nitratebilan_annuel_20220230.0
0730005072023nitratebilan_annuel_2023030.0
0690004372021nitratebilan_annuel_2021060.0
0670013302020nitratebilan_annuel_2020080.0
0890035032020nitratebilan_annuel_20203120.25
0640004162023nitratebilan_annuel_2023040.0
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+-----------+-------+-----------+-------------------+-----------------+-----------------+--------------------+\n", + "| cdreseau | annee | categorie | periode | nb_depassements | nb_prelevements | ratio_depassements |\n", + "+-----------+-------+-----------+-------------------+-----------------+-----------------+--------------------+\n", + "| 010000144 | 2024 | nitrate | bilan_annuel_2024 | 0 | 9 | 0.0 |\n", + "| 032000202 | 2023 | nitrate | bilan_annuel_2023 | 0 | 16 | 0.0 |\n", + "| 088001565 | 2020 | nitrate | bilan_annuel_2020 | 0 | 4 | 0.0 |\n", + "| 021000787 | 2023 | nitrate | bilan_annuel_2023 | 0 | 3 | 0.0 |\n", + "| 017000374 | 2022 | nitrate | bilan_annuel_2022 | 0 | 23 | 0.0 |\n", + "| 073000507 | 2023 | nitrate | bilan_annuel_2023 | 0 | 3 | 0.0 |\n", + "| 069000437 | 2021 | nitrate | bilan_annuel_2021 | 0 | 6 | 0.0 |\n", + "| 067001330 | 2020 | nitrate | bilan_annuel_2020 | 0 | 8 | 0.0 |\n", + "| 089003503 | 2020 | nitrate | bilan_annuel_2020 | 3 | 12 | 0.25 |\n", + "| 064000416 | 2023 | nitrate | bilan_annuel_2023 | 0 | 4 | 0.0 |\n", + "+-----------+-------+-----------+-------------------+-----------------+-----------------+--------------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --save int__resultats_nitrites_udi_annuel\n", + "WITH\n", + "prels AS (\n", + " SELECT\n", + " de_partition AS annee,\n", + " cdreseau,\n", + " referenceprel,\n", + " datetimeprel,\n", + " COUNT(DISTINCT cdparametresiseeaux) AS nb_parametres,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3_no2\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrate'\n", + " GROUP BY\n", + " annee,\n", + " cdreseau,\n", + " referenceprel,\n", + " datetimeprel,\n", + "),\n", + "\n", + "valeur_ref AS (\n", + " SELECT\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no2\n", + " FROM\n", + " int__valeurs_de_reference\n", + " WHERE\n", + " categorie_1 = 'nitrate'\n", + ")\n", + "\n", + "SELECT\n", + " prels.cdreseau,\n", + " prels.annee,\n", + " 'nitrate' AS categorie,\n", + " 'bilan_annuel_' || prels.annee AS periode,\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n", + " OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n", + " OR prels.valtraduite_no3_no2\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " THEN prels.referenceprel\n", + " END\n", + " ) AS nb_depassements,\n", + " COUNT(DISTINCT prels.referenceprel) AS nb_prelevements,\n", + " (\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n", + " OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n", + " OR prels.valtraduite_no3_no2\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " THEN prels.referenceprel\n", + " END\n", + " )::float\n", + " /\n", + " COUNT(DISTINCT prels.referenceprel)::float\n", + " ) AS ratio_depassements\n", + "\n", + "FROM prels\n", + "CROSS JOIN valeur_ref\n", + "GROUP BY prels.cdreseau, prels.annee\n" + ] + }, + { + "cell_type": "markdown", + "id": "c596b48d-ebc4-4fb9-aab8-b626076792d2", + "metadata": {}, + "source": [ + "# Check " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2137dd22-ec22-4b37-93ee-30b75c06ad42", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f8a18e6c7beb44c2a54490d921503406", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
periodecdreseaucategorieresultatnb_depassementsnb_prelevementsratio_depassements
" + ], + "text/plain": [ + "+---------+----------+-----------+----------+-----------------+-----------------+--------------------+\n", + "| periode | cdreseau | categorie | resultat | nb_depassements | nb_prelevements | ratio_depassements |\n", + "+---------+----------+-----------+----------+-----------------+-----------------+--------------------+\n", + "+---------+----------+-----------+----------+-----------------+-----------------+--------------------+" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_annuel \n", + "SELECT\n", + " 'annuel' AS periode,\n", + " cdreseau,\n", + " categorie,\n", + " '' AS resultat,\n", + " nb_depassements,\n", + " nb_prelevements,\n", + " ratio_depassements\n", + "FROM\n", + " int__resultats_nitrites_udi_annuel\n", + "WHERE\n", + " (\n", + " cdreseau = '092003070'\n", + " AND categorie = 'nitrate'\n", + " AND annee = '2024'\n", + " AND (nb_prelevements != 806 \n", + " OR nb_depassements != 0 )\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '071001155'\n", + " AND categorie = 'nitrate'\n", + " AND annee = '2023'\n", + " AND (nb_prelevements != 1 \n", + " OR nb_depassements != 0 )\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '036000670'\n", + " AND categorie = 'nitrate'\n", + " AND annee = '2024'\n", + " AND (nb_prelevements != 27 \n", + " OR nb_depassements != 27 )\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '092003070'\n", + " AND categorie = 'nitrate'\n", + " AND annee IN (2020, 2021, 2022, 2023, 2024, 2025) \n", + " AND nb_depassements != 0 \n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '089003503'\n", + " AND categorie = 'nitrate'\n", + " AND annee = '2020'\n", + " AND (nb_prelevements != 12 \n", + " OR nb_depassements != 3 )\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6dfd9bc4-176f-48ae-9db6-9b8be7b82b8d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 0eb2157a..0eada989 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -68,3 +68,62 @@ WHERE AND resultat != 'conforme' AND nb_parametres = 2 ) +UNION ALL +-- annuel +SELECT + 'annuel' AS periode, + cdreseau, + categorie, + '' AS resultat, + nb_depassements, + nb_prelevements, + ratio_depassements +FROM + {{ ref('int__resultats_nitrites_udi_annuel') }} +WHERE + ( + cdreseau = '092003070' + AND categorie = 'nitrate' + AND annee = '2024' + AND ( + nb_prelevements != 806 + OR nb_depassements != 0 + ) + ) + OR + ( + cdreseau = '071001155' + AND categorie = 'nitrate' + AND annee = '2023' + AND ( + nb_prelevements != 1 + OR nb_depassements != 0 + ) + ) + OR + ( + cdreseau = '036000670' + AND categorie = 'nitrate' + AND annee = '2024' + AND ( + nb_prelevements != 27 + OR nb_depassements != 27 + ) + ) + OR + ( + cdreseau = '092003070' + AND categorie = 'nitrate' + AND annee IN (2020, 2021, 2022, 2023, 2024, 2025) + AND nb_depassements != 0 + ) + OR + ( + cdreseau = '089003503' + AND categorie = 'nitrate' + AND annee = '2020' + AND ( + nb_prelevements != 12 + OR nb_depassements != 3 + ) + ) From 63f81e61c1c39aec415ed44e72ef917ea9ab6660 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sat, 12 Apr 2025 09:44:24 +0200 Subject: [PATCH 14/30] fix: valtraduite_no3_no2_cacl into split_nitrites_with_ref --- .../int__resultats_nitrites_udi_dernier.sql | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index b1f85552..93dc0487 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -91,10 +91,14 @@ split_nitrites_with_ref AS ( SELECT split_nitrites.*, valeur_ref.limite_qualite_no3, - valeur_ref.limite_qualite_no3_no2, valeur_ref.limite_qualite_no2, - split_nitrites.valtraduite_no3 / 50 - + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc + COALESCE( + valeur_ref.valtraduite_no3_no2, + ( + split_nitrites.valtraduite_no3 / 50 + + split_nitrites.valtraduite_no2 / 3 + ) + ) AS valtraduite_no3_no2 FROM split_nitrites CROSS JOIN valeur_ref @@ -112,10 +116,7 @@ SELECT < split_nitrites_with_ref.limite_qualite_no3 AND split_nitrites_with_ref.valtraduite_no2 < split_nitrites_with_ref.limite_qualite_no2 - AND COALESCE( - split_nitrites_with_ref.valtraduite_no3_no2, - split_nitrites_with_ref.valtraduite_no3_no2_calc - ) + AND split_nitrites_with_ref.valtraduite_no3_no2 < split_nitrites_with_ref.limite_qualite_no3_no2 THEN 'conforme' WHEN @@ -123,10 +124,7 @@ SELECT >= split_nitrites_with_ref.limite_qualite_no3 OR split_nitrites_with_ref.valtraduite_no2 >= split_nitrites_with_ref.limite_qualite_no2 - OR COALESCE( - split_nitrites_with_ref.valtraduite_no3_no2, - split_nitrites_with_ref.valtraduite_no3_no2_calc - ) + OR split_nitrites_with_ref.valtraduite_no3_no2 >= split_nitrites_with_ref.limite_qualite_no3_no2 THEN 'non_conforme' WHEN From f30fd22238fec99d4cbad489455355f76dfb4e03 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sat, 12 Apr 2025 10:54:20 +0200 Subject: [PATCH 15/30] feat: check date between test to define resultat --- analytics/notebooks/resultats_nitrates.ipynb | 1373 ++++++++++++++++- .../int__resultats_nitrites_udi_dernier.sql | 79 +- dbt_/tests/test_nitrates_results.sql | 32 + 3 files changed, 1401 insertions(+), 83 deletions(-) diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb index c571db67..7b3cb6aa 100644 --- a/analytics/notebooks/resultats_nitrates.ipynb +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -260,7 +260,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2abb38d323d04b04a1fcab8e91187bc0", + "model_id": "3637368299ca44af8a22405b94b25ae3", "version_major": 2, "version_minor": 0 }, @@ -287,82 +287,82 @@ " \n", " \n", " \n", - " 038000406\n", + " 001000254\n", " nitrate\n", " dernier_prel\n", - " 2024-11-04 10:30:00\n", + " 2024-12-26 10:15:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038000579\n", + " 001000285\n", " nitrate\n", " dernier_prel\n", - " 2024-10-08 10:45:00\n", + " 2025-01-22 11:24:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038000634\n", + " 001000289\n", " nitrate\n", " dernier_prel\n", - " 2025-01-30 09:20:00\n", + " 2025-01-21 11:14:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038000869\n", + " 001000359\n", " nitrate\n", " dernier_prel\n", - " 2024-10-29 08:45:00\n", + " 2025-01-13 10:42:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038000919\n", + " 001000369\n", " nitrate\n", " dernier_prel\n", - " 2025-02-27 09:48:00\n", + " 2025-02-19 09:12:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038000992\n", + " 001000408\n", " nitrate\n", " dernier_prel\n", - " 2025-01-02 10:35:00\n", + " 2024-07-17 08:33:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001014\n", + " 001000421\n", " nitrate\n", " dernier_prel\n", - " 2025-01-15 10:10:00\n", + " 2024-08-26 09:44:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001205\n", + " 001000493\n", " nitrate\n", " dernier_prel\n", - " 2024-11-14 09:35:00\n", + " 2024-09-13 11:08:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001237\n", + " 001000645\n", " nitrate\n", " dernier_prel\n", - " 2024-06-25 10:05:00\n", + " 2024-12-03 09:52:00\n", " 3\n", " conforme\n", " \n", " \n", - " 038001241\n", + " 001000715\n", " nitrate\n", " dernier_prel\n", - " 2024-12-19 09:30:00\n", + " 2024-08-26 11:52:00\n", " 3\n", " conforme\n", " \n", @@ -374,16 +374,16 @@ "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", - "| 038000406 | nitrate | dernier_prel | 2024-11-04 10:30:00 | 3 | conforme |\n", - "| 038000579 | nitrate | dernier_prel | 2024-10-08 10:45:00 | 3 | conforme |\n", - "| 038000634 | nitrate | dernier_prel | 2025-01-30 09:20:00 | 3 | conforme |\n", - "| 038000869 | nitrate | dernier_prel | 2024-10-29 08:45:00 | 3 | conforme |\n", - "| 038000919 | nitrate | dernier_prel | 2025-02-27 09:48:00 | 3 | conforme |\n", - "| 038000992 | nitrate | dernier_prel | 2025-01-02 10:35:00 | 3 | conforme |\n", - "| 038001014 | nitrate | dernier_prel | 2025-01-15 10:10:00 | 3 | conforme |\n", - "| 038001205 | nitrate | dernier_prel | 2024-11-14 09:35:00 | 3 | conforme |\n", - "| 038001237 | nitrate | dernier_prel | 2024-06-25 10:05:00 | 3 | conforme |\n", - "| 038001241 | nitrate | dernier_prel | 2024-12-19 09:30:00 | 3 | conforme |\n", + "| 001000254 | nitrate | dernier_prel | 2024-12-26 10:15:00 | 3 | conforme |\n", + "| 001000285 | nitrate | dernier_prel | 2025-01-22 11:24:00 | 3 | conforme |\n", + "| 001000289 | nitrate | dernier_prel | 2025-01-21 11:14:00 | 3 | conforme |\n", + "| 001000359 | nitrate | dernier_prel | 2025-01-13 10:42:00 | 3 | conforme |\n", + "| 001000369 | nitrate | dernier_prel | 2025-02-19 09:12:00 | 3 | conforme |\n", + "| 001000408 | nitrate | dernier_prel | 2024-07-17 08:33:00 | 3 | conforme |\n", + "| 001000421 | nitrate | dernier_prel | 2024-08-26 09:44:00 | 3 | conforme |\n", + "| 001000493 | nitrate | dernier_prel | 2024-09-13 11:08:00 | 3 | conforme |\n", + "| 001000645 | nitrate | dernier_prel | 2024-12-03 09:52:00 | 3 | conforme |\n", + "| 001000715 | nitrate | dernier_prel | 2024-08-26 11:52:00 | 3 | conforme |\n", "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", "Truncated to displaylimit of 10." ] @@ -561,7 +561,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7cef2c5b68ab42ad94b21ca3d9e681ac", + "model_id": "5059502bcb4f449e9f9fc1ec3cdd77e1", "version_major": 2, "version_minor": 0 }, @@ -584,16 +584,16 @@ " \n", " \n", " \n", + " donnee_manquante\n", + " 1162\n", + " \n", + " \n", " non_conforme\n", " 269\n", " \n", " \n", " conforme\n", - " 20741\n", - " \n", - " \n", - " donnee_manquante\n", - " 1149\n", + " 20704\n", " \n", " \n", "" @@ -602,9 +602,9 @@ "+------------------+--------------+\n", "| resultat | count_star() |\n", "+------------------+--------------+\n", + "| donnee_manquante | 1162 |\n", "| non_conforme | 269 |\n", - "| conforme | 20741 |\n", - "| donnee_manquante | 1149 |\n", + "| conforme | 20704 |\n", "+------------------+--------------+" ] }, @@ -643,7 +643,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cee2a9510a9748648016d352dee74573", + "model_id": "a1847396cf544b84862cf51d3514e1a9", "version_major": 2, "version_minor": 0 }, @@ -715,7 +715,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "53e139d8f8af457b8b9510adacb72410", + "model_id": "bcad3e0f29c643a494ac8920d4d1a5d1", "version_major": 2, "version_minor": 0 }, @@ -739,6 +739,11 @@ " \n", " \n", " \n", + " non_conforme\n", + " 002000060\n", + " 095000346\n", + " \n", + " \n", " conforme\n", " 001000003\n", " 976003489\n", @@ -748,11 +753,6 @@ " 001000251\n", " 976003881\n", " \n", - " \n", - " non_conforme\n", - " 002000060\n", - " 095000346\n", - " \n", " \n", "" ], @@ -760,9 +760,9 @@ "+------------------+--------------+--------------+\n", "| resultat | min_cdreseau | max_cdreseau |\n", "+------------------+--------------+--------------+\n", + "| non_conforme | 002000060 | 095000346 |\n", "| conforme | 001000003 | 976003489 |\n", "| donnee_manquante | 001000251 | 976003881 |\n", - "| non_conforme | 002000060 | 095000346 |\n", "+------------------+--------------+--------------+" ] }, @@ -809,7 +809,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d2e860daff024a4491bcec409b322be2", + "model_id": "ef9a50f69b794ae9862fec9d38d85ac7", "version_major": 2, "version_minor": 0 }, @@ -835,6 +835,7 @@ " valtraduite_no3\n", " valtraduite_no3_no2\n", " valtraduite_no3_no2_calc\n", + " date_diff_count\n", " \n", " \n", " \n", @@ -848,6 +849,7 @@ " 14.000\n", " 0.280\n", " 0.28\n", + " 0\n", " \n", " \n", " 001000260\n", @@ -859,6 +861,7 @@ " 2.400\n", " 0.050\n", " 0.048\n", + " 0\n", " \n", " \n", " 001000268\n", @@ -870,6 +873,7 @@ " 6.000\n", " 0.120\n", " 0.12\n", + " 0\n", " \n", " \n", " 001000293\n", @@ -881,6 +885,7 @@ " 3.700\n", " 0.070\n", " 0.07400000000000001\n", + " 0\n", " \n", " \n", " 002000060\n", @@ -892,6 +897,7 @@ " 59.300\n", " 0.000\n", " 1.186\n", + " 0\n", " \n", " \n", " 073002059\n", @@ -903,6 +909,7 @@ " None\n", " None\n", " None\n", + " 0\n", " \n", " \n", " 088002296\n", @@ -914,6 +921,7 @@ " None\n", " None\n", " None\n", + " 0\n", " \n", " \n", " 095000346\n", @@ -925,6 +933,7 @@ " 47.000\n", " 1.000\n", " 0.94\n", + " 241\n", " \n", " \n", " 973000028\n", @@ -936,24 +945,25 @@ " 0.000\n", " None\n", " 0.0\n", + " 0\n", " \n", " \n", "" ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | valtraduite_no2 | valtraduite_no3 | valtraduite_no3_no2 | valtraduite_no3_no2_calc |\n", - "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+\n", - "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | 0.000 | 14.000 | 0.280 | 0.28 |\n", - "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | 0.000 | 2.400 | 0.050 | 0.048 |\n", - "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | 0.000 | 6.000 | 0.120 | 0.12 |\n", - "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | 0.000 | 3.700 | 0.070 | 0.07400000000000001 |\n", - "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | 0.000 | 59.300 | 0.000 | 1.186 |\n", - "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | 0.000 | None | None | None |\n", - "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | 0.000 | None | None | None |\n", - "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | 0.000 | 47.000 | 1.000 | 0.94 |\n", - "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | 0.000 | 0.000 | None | 0.0 |\n", - "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+" + "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+-----------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | valtraduite_no2 | valtraduite_no3 | valtraduite_no3_no2 | valtraduite_no3_no2_calc | date_diff_count |\n", + "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+-----------------+\n", + "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | 0.000 | 14.000 | 0.280 | 0.28 | 0 |\n", + "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | 0.000 | 2.400 | 0.050 | 0.048 | 0 |\n", + "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | 0.000 | 6.000 | 0.120 | 0.12 | 0 |\n", + "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | 0.000 | 3.700 | 0.070 | 0.07400000000000001 | 0 |\n", + "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | 0.000 | 59.300 | 0.000 | 1.186 | 0 |\n", + "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | 0.000 | None | None | None | 0 |\n", + "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | 0.000 | None | None | None | 0 |\n", + "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | 0.000 | 47.000 | 1.000 | 0.94 | 241 |\n", + "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | 0.000 | 0.000 | None | 0.0 | 0 |\n", + "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+-----------------+" ] }, "execution_count": 8, @@ -1078,7 +1088,20 @@ " valeur_ref.limite_qualite_no3_no2,\n", " valeur_ref.limite_qualite_no2,\n", " split_nitrites.valtraduite_no3 / 50\n", - " + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc\n", + " + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc,\n", + " DATE_DIFF(\n", + " 'day',\n", + " LEAST(\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02,\n", + " split_nitrites.dernier_prel_datetime_no3_no2\n", + " ),\n", + " GREATEST(\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02,\n", + " split_nitrites.dernier_prel_datetime_no3_no2\n", + " )\n", + " ) AS date_diff_count\n", " FROM split_nitrites\n", " CROSS JOIN valeur_ref\n", "\n", @@ -1090,10 +1113,11 @@ " 'dernier_prel' AS periode,\n", " split_nitrites_with_ref.dernier_prel_datetime,\n", " split_nitrites_with_ref.nb_parametres,\n", - " split_nitrites_with_ref.valtraduite_no2,\n", - " split_nitrites_with_ref.valtraduite_no3,\n", - " split_nitrites_with_ref.valtraduite_no3_no2,\n", - " split_nitrites_with_ref.valtraduite_no3_no2_calc\n", + " split_nitrites_with_ref.valtraduite_no2,\n", + " split_nitrites_with_ref.valtraduite_no3,\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " split_nitrites_with_ref.valtraduite_no3_no2_calc,\n", + " split_nitrites_with_ref.date_diff_count\n", "FROM\n", " split_nitrites_with_ref \n", "WHERE\n", @@ -1123,7 +1147,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9249792e8cbd438ab76d46ddb8001245", + "model_id": "dcbe2dd72bd94fe4ad82115686a395eb", "version_major": 2, "version_minor": 0 }, @@ -1257,10 +1281,1223 @@ " cdreseau" ] }, + { + "cell_type": "markdown", + "id": "2cee74c0-c1e2-4b4e-9ccc-aef10805f621", + "metadata": {}, + "source": [ + "# Ajout d'une condition sur les dates de prélèvement/mesure\n", + "\n", + "**Les règles choisies**\n", + " \n", + "_Si on a les 3 paramètres disponibles **avec moins de 30 jours de différences**_\n", + "- Si Nitrates < 50 mg/L et nitrites < 0,5 mg/L et nitrate/50 + nitrites/3 < 1 mg/L --> CONFORME\n", + "- Si Nitrates >= 50 mg/L et/ou nitrites >= 0,5 mg/L et/ou nitrate/50 + nitrites/3 >= 1 mg/L --> NON CONFORME\n", + "\n", + "\n", + "_Si on n'a pas nitrate/50 + nitrites/3 mais qu'on a nitrate et nitrites_
\n", + "==> on le calcule et on applique comme ci-dessus\n", + "\n", + "- Si on n'a pas nitrate (no3) --> donnee_manquante\n", + "- Si on n'a pas nitrites (no2) et que nitrate >= 50 mg/L --> NON CONFORME\n", + "- Si on n'a pas nitrites (no2) et que nitrate < 50 mg/L --> donnee_manquant \n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "27337155-ed39-4db0-931a-197c9c055538", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "90c402d060c64f848bc8868c9b5ae212", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
date_diffdernier_prel_datetime_n03dernier_prel_datetime_n02cdreseaucategorienb_parametresdernier_prel_datetimevaltraduite_no3valtraduite_no2valtraduite_no3_no2date_diff_count
-452024-11-25 09:29:002024-10-11 12:55:00002000074nitrate32024-11-25 09:29:0056.900None0.045
-612024-12-23 13:09:002024-10-23 08:29:00002001241nitrate32024-12-23 13:09:0052.500None0.061
-1412025-01-28 09:24:002024-09-09 09:09:00008000002nitrate22025-01-28 09:24:0052.000None1.04141
-1612025-02-07 10:38:002024-08-30 13:21:00008000546nitrate22025-02-07 10:38:0076.300None1.526161
-372025-02-26 11:59:002025-01-20 13:30:00008000679nitrate32025-02-26 11:59:0053.700None0.037
-1002025-02-12 11:38:002024-11-04 11:23:00008000681nitrate22025-02-12 11:38:0054.900None1.0979999999999999100
-1422025-02-19 12:10:002024-09-30 08:32:00008001171nitrate22025-02-19 12:10:0050.500None1.01142
-742025-02-18 10:16:002024-12-06 08:10:00010000072nitrate32025-02-18 10:16:0056.000None1.174
-1752025-02-18 12:24:002024-08-27 10:03:00010000122nitrate32025-02-18 12:24:0064.000None1.1175
-982025-02-24 11:25:002024-11-18 11:22:00010000125nitrate32025-02-24 11:25:0072.000None1.498
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+-----------+---------------------------+---------------------------+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+-----------------+\n", + "| date_diff | dernier_prel_datetime_n03 | dernier_prel_datetime_n02 | cdreseau | categorie | nb_parametres | dernier_prel_datetime | valtraduite_no3 | valtraduite_no2 | valtraduite_no3_no2 | date_diff_count |\n", + "+-----------+---------------------------+---------------------------+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+-----------------+\n", + "| -45 | 2024-11-25 09:29:00 | 2024-10-11 12:55:00 | 002000074 | nitrate | 3 | 2024-11-25 09:29:00 | 56.900 | None | 0.0 | 45 |\n", + "| -61 | 2024-12-23 13:09:00 | 2024-10-23 08:29:00 | 002001241 | nitrate | 3 | 2024-12-23 13:09:00 | 52.500 | None | 0.0 | 61 |\n", + "| -141 | 2025-01-28 09:24:00 | 2024-09-09 09:09:00 | 008000002 | nitrate | 2 | 2025-01-28 09:24:00 | 52.000 | None | 1.04 | 141 |\n", + "| -161 | 2025-02-07 10:38:00 | 2024-08-30 13:21:00 | 008000546 | nitrate | 2 | 2025-02-07 10:38:00 | 76.300 | None | 1.526 | 161 |\n", + "| -37 | 2025-02-26 11:59:00 | 2025-01-20 13:30:00 | 008000679 | nitrate | 3 | 2025-02-26 11:59:00 | 53.700 | None | 0.0 | 37 |\n", + "| -100 | 2025-02-12 11:38:00 | 2024-11-04 11:23:00 | 008000681 | nitrate | 2 | 2025-02-12 11:38:00 | 54.900 | None | 1.0979999999999999 | 100 |\n", + "| -142 | 2025-02-19 12:10:00 | 2024-09-30 08:32:00 | 008001171 | nitrate | 2 | 2025-02-19 12:10:00 | 50.500 | None | 1.01 | 142 |\n", + "| -74 | 2025-02-18 10:16:00 | 2024-12-06 08:10:00 | 010000072 | nitrate | 3 | 2025-02-18 10:16:00 | 56.000 | None | 1.1 | 74 |\n", + "| -175 | 2025-02-18 12:24:00 | 2024-08-27 10:03:00 | 010000122 | nitrate | 3 | 2025-02-18 12:24:00 | 64.000 | None | 1.1 | 175 |\n", + "| -98 | 2025-02-24 11:25:00 | 2024-11-18 11:22:00 | 010000125 | nitrate | 3 | 2025-02-24 11:25:00 | 72.000 | None | 1.4 | 98 |\n", + "+-----------+---------------------------+---------------------------+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+-----------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "WITH\n", + "last_pvl AS (\n", + " SELECT\n", + " cdreseau,\n", + " categorie,\n", + " cdparametresiseeaux,\n", + " limite_qualite,\n", + " datetimeprel,\n", + " valtraduite,\n", + " ROW_NUMBER()\n", + " OVER (\n", + " PARTITION BY cdreseau, cdparametresiseeaux\n", + " ORDER BY datetimeprel DESC\n", + " )\n", + " AS row_number\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrate'\n", + " AND\n", + " -- On garde les prélèvements de moins d'un an\n", + " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + "),\n", + "\n", + "valeur_ref AS (\n", + "SELECT \n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no2\n", + "FROM int__valeurs_de_reference\n", + "WHERE categorie_1='nitrate' \n", + "),\n", + "\n", + "split_nitrites AS (\n", + " SELECT\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie,\n", + " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", + " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n03,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n02,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.valtraduite\n", + " END\n", + " ) AS valtraduite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.valtraduite\n", + " END\n", + " ) AS valtraduite_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN last_pvl.valtraduite\n", + " END\n", + " ) AS valtraduite_no3_no2\n", + " FROM\n", + " last_pvl\n", + " WHERE\n", + " last_pvl.row_number = 1\n", + " GROUP BY\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie\n", + "),\n", + "\n", + "split_nitrites_with_ref AS (\n", + " SELECT \n", + " DATE_DIFF( 'day',split_nitrites.dernier_prel_datetime_n03,split_nitrites.dernier_prel_datetime_n02) AS date_diff,\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02,\n", + " split_nitrites.cdreseau,\n", + " split_nitrites.categorie,\n", + " split_nitrites.nb_parametres,\n", + " split_nitrites.dernier_prel_datetime,\n", + " IF ( DATE_DIFF( 'day',split_nitrites.dernier_prel_datetime_n03,split_nitrites.dernier_prel_datetime_n02) >30,\n", + " NULL, valtraduite_no3) AS valtraduite_no3,\n", + " IF ( DATE_DIFF( 'day',split_nitrites.dernier_prel_datetime_n02,split_nitrites.dernier_prel_datetime_n03) >30,\n", + " NULL, valtraduite_no2) AS valtraduite_no2,\n", + " COALESCE(\n", + " split_nitrites.valtraduite_no3_no2,\n", + " (\n", + " split_nitrites.valtraduite_no3 / 50\n", + " + split_nitrites.valtraduite_no2 / 3\n", + " )\n", + " ) AS valtraduite_no3_no2,\n", + " \n", + " DATE_DIFF(\n", + " 'day',\n", + " LEAST(\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02\n", + " ),\n", + " GREATEST(\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02\n", + " )\n", + " ) AS date_diff_count\n", + " FROM split_nitrites\n", + " CROSS JOIN valeur_ref\n", + "\n", + ")\n", + "\n", + "SELECT\n", + "*\n", + "FROM\n", + " split_nitrites_with_ref \n", + "WHERE\n", + " split_nitrites_with_ref.date_diff_count > 30\n", + " AND valtraduite_no3 > 50\n", + "ORDER BY \n", + " cdreseau" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "dbdcec00-8e5c-4eb0-9fac-2a5f90709383", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ee1b9b581f6948db80e00512efe67a8f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000241nitratedernier_prel2025-02-10 11:04:003conforme
001000304nitratedernier_prel2025-02-11 10:10:003conforme
001000313nitratedernier_prel2024-11-27 11:00:003conforme
001000387nitratedernier_prel2025-02-13 10:48:003donnee_manquante
001000427nitratedernier_prel2025-01-24 11:15:003conforme
001000464nitratedernier_prel2025-01-16 09:17:003conforme
001000490nitratedernier_prel2024-09-27 11:53:003conforme
001000585nitratedernier_prel2025-02-21 09:15:003conforme
001000824nitratedernier_prel2024-10-18 09:54:003conforme
001000865nitratedernier_prel2025-02-11 09:27:003conforme
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| 001000241 | nitrate | dernier_prel | 2025-02-10 11:04:00 | 3 | conforme |\n", + "| 001000304 | nitrate | dernier_prel | 2025-02-11 10:10:00 | 3 | conforme |\n", + "| 001000313 | nitrate | dernier_prel | 2024-11-27 11:00:00 | 3 | conforme |\n", + "| 001000387 | nitrate | dernier_prel | 2025-02-13 10:48:00 | 3 | donnee_manquante |\n", + "| 001000427 | nitrate | dernier_prel | 2025-01-24 11:15:00 | 3 | conforme |\n", + "| 001000464 | nitrate | dernier_prel | 2025-01-16 09:17:00 | 3 | conforme |\n", + "| 001000490 | nitrate | dernier_prel | 2024-09-27 11:53:00 | 3 | conforme |\n", + "| 001000585 | nitrate | dernier_prel | 2025-02-21 09:15:00 | 3 | conforme |\n", + "| 001000824 | nitrate | dernier_prel | 2024-10-18 09:54:00 | 3 | conforme |\n", + "| 001000865 | nitrate | dernier_prel | 2025-02-11 09:27:00 | 3 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --save int__resultats_nitrites_udi_dernier_dateCheck\n", + "WITH\n", + "last_pvl AS (\n", + " SELECT\n", + " cdreseau,\n", + " categorie,\n", + " cdparametresiseeaux,\n", + " limite_qualite,\n", + " datetimeprel,\n", + " valtraduite,\n", + " ROW_NUMBER()\n", + " OVER (\n", + " PARTITION BY cdreseau, cdparametresiseeaux\n", + " ORDER BY datetimeprel DESC\n", + " )\n", + " AS row_number\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrate'\n", + " AND\n", + " -- On garde les prélèvements de moins d'un an\n", + " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + "),\n", + "\n", + "valeur_ref AS (\n", + " SELECT\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no2\n", + " FROM\n", + " int__valeurs_de_reference\n", + " WHERE\n", + " categorie_1 = 'nitrate'\n", + "),\n", + "\n", + "split_nitrites AS (\n", + " SELECT\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie,\n", + " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", + " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.valtraduite\n", + " END\n", + " ) AS valtraduite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.valtraduite\n", + " END\n", + " ) AS valtraduite_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN last_pvl.valtraduite\n", + " END\n", + " ) AS valtraduite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO3'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n03,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " last_pvl.cdparametresiseeaux = 'NO2'\n", + " THEN last_pvl.datetimeprel\n", + " END\n", + " ) AS dernier_prel_datetime_n02\n", + " FROM\n", + " last_pvl\n", + " WHERE\n", + " last_pvl.row_number = 1\n", + " GROUP BY\n", + " last_pvl.cdreseau,\n", + " last_pvl.categorie\n", + "),\n", + "\n", + "split_nitrites_with_ref AS (\n", + " SELECT\n", + " split_nitrites.cdreseau,\n", + " split_nitrites.categorie,\n", + " split_nitrites.nb_parametres,\n", + " split_nitrites.dernier_prel_datetime,\n", + " valeur_ref.limite_qualite_no3,\n", + " valeur_ref.limite_qualite_no2,\n", + " valeur_ref.limite_qualite_no3_no2,\n", + " split_nitrites.valtraduite_no3_no2,\n", + " -- Si les mesures sont espacées de plus de 30 jours \n", + " -- on garde que la plus récente uniquement \n", + " IF(\n", + " DATE_DIFF(\n", + " 'day',\n", + " split_nitrites.dernier_prel_datetime_n03,\n", + " split_nitrites.dernier_prel_datetime_n02\n", + " ) > 30,\n", + " NULL,\n", + " split_nitrites.valtraduite_no3\n", + " ) AS valtraduite_no3,\n", + " IF(\n", + " DATE_DIFF(\n", + " 'day',\n", + " split_nitrites.dernier_prel_datetime_n02,\n", + " split_nitrites.dernier_prel_datetime_n03\n", + " ) > 30,\n", + " NULL,\n", + " split_nitrites.valtraduite_no2\n", + " ) AS valtraduite_no2\n", + " FROM\n", + " split_nitrites\n", + " CROSS JOIN\n", + " valeur_ref\n", + ")\n", + "\n", + "SELECT\n", + " split_nitrites_with_ref.cdreseau,\n", + " split_nitrites_with_ref.categorie,\n", + " 'dernier_prel' AS periode,\n", + " split_nitrites_with_ref.dernier_prel_datetime,\n", + " split_nitrites_with_ref.nb_parametres,\n", + " CASE\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", + " AND split_nitrites_with_ref.valtraduite_no2\n", + " < split_nitrites_with_ref.limite_qualite_no2\n", + " -- Si on n'a pas valtraduite_no3_no2, on le calcule\n", + " -- si l'une des valeurs de la somme est NULL le total sera NULL \n", + " AND COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " (\n", + " split_nitrites_with_ref.valtraduite_no3 / 50\n", + " + split_nitrites_with_ref.valtraduite_no2 / 3\n", + " )\n", + " )\n", + " < split_nitrites_with_ref.limite_qualite_no3_no2\n", + " THEN 'conforme'\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " >= split_nitrites_with_ref.limite_qualite_no3\n", + " OR split_nitrites_with_ref.valtraduite_no2\n", + " >= split_nitrites_with_ref.limite_qualite_no2\n", + " -- Si on n'a pas valtraduite_no3_no2, on le calcule\n", + " -- si l'une des valeurs de la somme est NULL le total sera NULL \n", + " OR COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " (\n", + " split_nitrites_with_ref.valtraduite_no3 / 50\n", + " + split_nitrites_with_ref.valtraduite_no2 / 3\n", + " )\n", + " )\n", + " >= split_nitrites_with_ref.limite_qualite_no3_no2\n", + " THEN 'non_conforme'\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3 IS NULL\n", + " OR (\n", + " split_nitrites_with_ref.valtraduite_no2 IS NULL\n", + " AND split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", + " )\n", + " THEN 'donnee_manquante'\n", + " ELSE 'error'\n", + " END AS resultat\n", + "FROM\n", + " split_nitrites_with_ref\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3f32e544-93cb-4728-8015-b3183ec50f44", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "04a10f29f2204ff38289c79a1b04b10c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
resultatcount_star()
non_conforme269
conforme17338
donnee_manquante4528
" + ], + "text/plain": [ + "+------------------+--------------+\n", + "| resultat | count_star() |\n", + "+------------------+--------------+\n", + "| non_conforme | 269 |\n", + "| conforme | 17338 |\n", + "| donnee_manquante | 4528 |\n", + "+------------------+--------------+" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck\n", + "SELECT \n", + " resultat, COUNT(*)\n", + "FROM \n", + " int__resultats_nitrites_udi_dernier_dateCheck\n", + "GROUP BY 1" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1c02a4fb-edb6-4822-8a24-cedb82705f45", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6db26e98b8ee494aa91c4190644dcd44", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
" + ], + "text/plain": [ + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "+----------+-----------+---------+-----------------------+---------------+----------+" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck\n", + "SELECT \n", + " * \n", + "FROM \n", + " int__resultats_nitrites_udi_dernier_dateCheck\n", + "WHERE \n", + " resultat='error'" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "674c3c38-693f-4025-86e4-554627d568ad", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9f02b918c1054b138d148edd1a0ebc21", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000003nitratedernier_prel2025-01-21 12:35:003conforme
001000258nitratedernier_prel2025-01-21 09:45:003donnee_manquante
001000260nitratedernier_prel2024-12-06 12:02:003conforme
001000268nitratedernier_prel2025-01-13 09:57:003conforme
001000293nitratedernier_prel2025-01-28 11:29:003conforme
001000387nitratedernier_prel2025-02-13 10:48:003donnee_manquante
002000060nitratedernier_prel2024-10-18 08:50:003non_conforme
008000546nitratedernier_prel2025-02-07 10:38:002non_conforme
008000681nitratedernier_prel2025-02-12 11:38:002non_conforme
073002059nitratedernier_prel2024-10-23 09:28:001donnee_manquante
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | conforme |\n", + "| 001000258 | nitrate | dernier_prel | 2025-01-21 09:45:00 | 3 | donnee_manquante |\n", + "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", + "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | conforme |\n", + "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | conforme |\n", + "| 001000387 | nitrate | dernier_prel | 2025-02-13 10:48:00 | 3 | donnee_manquante |\n", + "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", + "| 008000546 | nitrate | dernier_prel | 2025-02-07 10:38:00 | 2 | non_conforme |\n", + "| 008000681 | nitrate | dernier_prel | 2025-02-12 11:38:00 | 2 | non_conforme |\n", + "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", + " SELECT \n", + " *\n", + " FROM\n", + " int__resultats_nitrites_udi_dernier_dateCheck\n", + "WHERE\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293',\n", + " '001000258','001000387','008000681','008000546')\n", + "ORDER BY \n", + " cdreseau" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "4033d129-eeab-4ec9-ac40-39a4101e3083", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7e1709e2f4e34e068d5a3fe1ea29ce6d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
periodecdreseaucategorieresultat
" + ], + "text/plain": [ + "+---------+----------+-----------+----------+\n", + "| periode | cdreseau | categorie | resultat |\n", + "+---------+----------+-----------+----------+\n", + "+---------+----------+-----------+----------+" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", + "SELECT\n", + " 'dernier relevé' AS periode,\n", + " cdreseau,\n", + " categorie,\n", + " resultat,\n", + "FROM\n", + " int__resultats_nitrites_udi_dernier_dateCheck\n", + "WHERE\n", + " (\n", + " cdreseau = '001000003'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-12-26 09:08:00'\n", + " AND resultat != 'conforme'\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '001000260'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-12-06 12:02:00'\n", + " AND resultat != 'conforme'\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '001000268'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-11-28 08:25:00'\n", + " AND resultat != 'conforme'\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '002000060'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-10-18 08:50:00'\n", + " AND resultat != 'non_conforme'\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '073002059'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-10-23 09:28:00'\n", + " AND resultat != 'donnee_manquante'\n", + " AND nb_parametres = 1\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '088002296'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-09-27 12:00:00'\n", + " AND resultat != 'donnee_manquante'\n", + " AND nb_parametres = 1\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '095000346'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-12-17 08:55:00'\n", + " AND resultat != 'non_conforme'\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '973000028'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2024-12-17 08:55:00'\n", + " AND resultat != 'conforme'\n", + " AND nb_parametres = 2\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '001000258'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2025-01-21 09:45:00'\n", + " AND resultat != 'donnee_manquante'\n", + " AND nb_parametres = 2\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '001000387'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2025-02-13 10:48:00'\n", + " AND resultat != 'conforme'\n", + " AND nb_parametres = 2\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '008000681'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2025-02-12 11:38:00'\n", + " AND resultat != 'non_conforme'\n", + " AND nb_parametres = 2\n", + " )\n", + " OR\n", + " (\n", + " cdreseau = '008000546'\n", + " AND categorie = 'nitrate'\n", + " AND dernier_prel_datetime = '2025-02-07 10:38:00'\n", + " AND resultat != 'non_conforme'\n", + " AND nb_parametres = 2\n", + " )" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "6dfd9bc4-176f-48ae-9db6-9b8be7b82b8d", + "id": "e5c04a09-1064-4041-8d2a-3d662202801a", "metadata": {}, "outputs": [], "source": [] diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index 93dc0487..a632daf1 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -46,7 +46,7 @@ valeur_ref AS ( END ) AS limite_qualite_no2 FROM - int__valeurs_de_reference + {{ ref('int__valeurs_de_reference') }} WHERE categorie_1 = 'nitrate' ), @@ -77,7 +77,21 @@ split_nitrites AS ( last_pvl.cdparametresiseeaux = 'NO3_NO2' THEN last_pvl.valtraduite END - ) AS valtraduite_no3_no2 + ) AS valtraduite_no3_no2, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO3' + THEN last_pvl.datetimeprel + END + ) AS dernier_prel_datetime_n03, + MAX( + CASE + WHEN + last_pvl.cdparametresiseeaux = 'NO2' + THEN last_pvl.datetimeprel + END + ) AS dernier_prel_datetime_n02 FROM last_pvl WHERE @@ -89,19 +103,38 @@ split_nitrites AS ( split_nitrites_with_ref AS ( SELECT - split_nitrites.*, + split_nitrites.cdreseau, + split_nitrites.categorie, + split_nitrites.nb_parametres, + split_nitrites.dernier_prel_datetime, valeur_ref.limite_qualite_no3, valeur_ref.limite_qualite_no2, - COALESCE( - valeur_ref.valtraduite_no3_no2, - ( - split_nitrites.valtraduite_no3 / 50 - + split_nitrites.valtraduite_no2 / 3 - ) - ) AS valtraduite_no3_no2 - FROM split_nitrites - CROSS JOIN valeur_ref - + valeur_ref.limite_qualite_no3_no2, + split_nitrites.valtraduite_no3_no2, + -- Si les mesures sont espacées de plus de 30 jours + -- on garde que la plus récente uniquement + IF( + DATE_DIFF( + 'day', + split_nitrites.dernier_prel_datetime_n03, + split_nitrites.dernier_prel_datetime_n02 + ) > 30, + NULL, + split_nitrites.valtraduite_no3 + ) AS valtraduite_no3, + IF( + DATE_DIFF( + 'day', + split_nitrites.dernier_prel_datetime_n02, + split_nitrites.dernier_prel_datetime_n03 + ) > 30, + NULL, + split_nitrites.valtraduite_no2 + ) AS valtraduite_no2 + FROM + split_nitrites + CROSS JOIN + valeur_ref ) SELECT @@ -116,7 +149,15 @@ SELECT < split_nitrites_with_ref.limite_qualite_no3 AND split_nitrites_with_ref.valtraduite_no2 < split_nitrites_with_ref.limite_qualite_no2 - AND split_nitrites_with_ref.valtraduite_no3_no2 + -- Si on n'a pas valtraduite_no3_no2, on le calcule + -- si l'une des valeurs de la somme est NULL le total sera NULL + AND COALESCE( + split_nitrites_with_ref.valtraduite_no3_no2, + ( + split_nitrites_with_ref.valtraduite_no3 / 50 + + split_nitrites_with_ref.valtraduite_no2 / 3 + ) + ) < split_nitrites_with_ref.limite_qualite_no3_no2 THEN 'conforme' WHEN @@ -124,7 +165,15 @@ SELECT >= split_nitrites_with_ref.limite_qualite_no3 OR split_nitrites_with_ref.valtraduite_no2 >= split_nitrites_with_ref.limite_qualite_no2 - OR split_nitrites_with_ref.valtraduite_no3_no2 + -- Si on n'a pas valtraduite_no3_no2, on le calcule + -- si l'une des valeurs de la somme est NULL le total sera NULL + OR COALESCE( + split_nitrites_with_ref.valtraduite_no3_no2, + ( + split_nitrites_with_ref.valtraduite_no3 / 50 + + split_nitrites_with_ref.valtraduite_no2 / 3 + ) + ) >= split_nitrites_with_ref.limite_qualite_no3_no2 THEN 'non_conforme' WHEN diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 0eb2157a..84eb8a23 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -68,3 +68,35 @@ WHERE AND resultat != 'conforme' AND nb_parametres = 2 ) + OR + ( + cdreseau = '001000258' + AND categorie = 'nitrate' + AND dernier_prel_datetime = '2025-01-21 09:45:00' + AND resultat != 'donnee_manquante' + AND nb_parametres = 2 + ) + OR + ( + cdreseau = '001000387' + AND categorie = 'nitrate' + AND dernier_prel_datetime = '2025-02-13 10:48:00' + AND resultat != 'conforme' + AND nb_parametres = 2 + ) + OR + ( + cdreseau = '008000681' + AND categorie = 'nitrate' + AND dernier_prel_datetime = '2025-02-12 11:38:00' + AND resultat != 'non_conforme' + AND nb_parametres = 2 + ) + OR + ( + cdreseau = '008000546' + AND categorie = 'nitrate' + AND dernier_prel_datetime = '2025-02-07 10:38:00' + AND resultat != 'non_conforme' + AND nb_parametres = 2 + ) \ No newline at end of file From 33a9e647cf1c724dc308a0fd039523d6d5f1916e Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sat, 12 Apr 2025 11:04:39 +0200 Subject: [PATCH 16/30] fix: pre-commit --- dbt_/tests/test_nitrates_results.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 84eb8a23..1b24efd9 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -99,4 +99,4 @@ WHERE AND dernier_prel_datetime = '2025-02-07 10:38:00' AND resultat != 'non_conforme' AND nb_parametres = 2 - ) \ No newline at end of file + ) From 2e2ef7b4904f5ebe00a759dbfd16e1a7ce4815f3 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Sun, 13 Apr 2025 20:54:24 +0200 Subject: [PATCH 17/30] fix: update test --- analytics/notebooks/resultats_nitrates.ipynb | 414 ++++++++++++------- dbt_/tests/test_nitrates_results.sql | 17 +- 2 files changed, 262 insertions(+), 169 deletions(-) diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb index 7b3cb6aa..836b8eb9 100644 --- a/analytics/notebooks/resultats_nitrates.ipynb +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -260,7 +260,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3637368299ca44af8a22405b94b25ae3", + "model_id": "c7b7f35ae9004e30bdf76cb212cb50ee", "version_major": 2, "version_minor": 0 }, @@ -287,82 +287,82 @@ " \n", " \n", " \n", - " 001000254\n", + " 062000617\n", " nitrate\n", " dernier_prel\n", - " 2024-12-26 10:15:00\n", + " 2025-02-07 11:19:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000285\n", + " 062000620\n", " nitrate\n", " dernier_prel\n", - " 2025-01-22 11:24:00\n", - " 3\n", + " 2024-09-25 11:59:00\n", + " 2\n", " conforme\n", " \n", " \n", - " 001000289\n", + " 062000655\n", " nitrate\n", " dernier_prel\n", - " 2025-01-21 11:14:00\n", + " 2024-10-03 11:16:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000359\n", + " 062000660\n", " nitrate\n", " dernier_prel\n", - " 2025-01-13 10:42:00\n", + " 2025-02-13 08:57:00\n", " 3\n", - " conforme\n", + " non_conforme\n", " \n", " \n", - " 001000369\n", + " 062000735\n", " nitrate\n", " dernier_prel\n", - " 2025-02-19 09:12:00\n", + " 2025-02-24 09:23:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000408\n", + " 062000738\n", " nitrate\n", " dernier_prel\n", - " 2024-07-17 08:33:00\n", + " 2025-02-12 10:08:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000421\n", + " 062000769\n", " nitrate\n", " dernier_prel\n", - " 2024-08-26 09:44:00\n", + " 2025-01-13 10:26:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000493\n", + " 062000776\n", " nitrate\n", " dernier_prel\n", - " 2024-09-13 11:08:00\n", + " 2024-11-25 08:22:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000645\n", + " 062000830\n", " nitrate\n", " dernier_prel\n", - " 2024-12-03 09:52:00\n", + " 2025-02-25 12:33:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000715\n", + " 062000863\n", " nitrate\n", " dernier_prel\n", - " 2024-08-26 11:52:00\n", + " 2025-01-07 08:40:00\n", " 3\n", " conforme\n", " \n", @@ -371,20 +371,20 @@ "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", - "| 001000254 | nitrate | dernier_prel | 2024-12-26 10:15:00 | 3 | conforme |\n", - "| 001000285 | nitrate | dernier_prel | 2025-01-22 11:24:00 | 3 | conforme |\n", - "| 001000289 | nitrate | dernier_prel | 2025-01-21 11:14:00 | 3 | conforme |\n", - "| 001000359 | nitrate | dernier_prel | 2025-01-13 10:42:00 | 3 | conforme |\n", - "| 001000369 | nitrate | dernier_prel | 2025-02-19 09:12:00 | 3 | conforme |\n", - "| 001000408 | nitrate | dernier_prel | 2024-07-17 08:33:00 | 3 | conforme |\n", - "| 001000421 | nitrate | dernier_prel | 2024-08-26 09:44:00 | 3 | conforme |\n", - "| 001000493 | nitrate | dernier_prel | 2024-09-13 11:08:00 | 3 | conforme |\n", - "| 001000645 | nitrate | dernier_prel | 2024-12-03 09:52:00 | 3 | conforme |\n", - "| 001000715 | nitrate | dernier_prel | 2024-08-26 11:52:00 | 3 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+----------+\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------+\n", + "| 062000617 | nitrate | dernier_prel | 2025-02-07 11:19:00 | 3 | conforme |\n", + "| 062000620 | nitrate | dernier_prel | 2024-09-25 11:59:00 | 2 | conforme |\n", + "| 062000655 | nitrate | dernier_prel | 2024-10-03 11:16:00 | 3 | conforme |\n", + "| 062000660 | nitrate | dernier_prel | 2025-02-13 08:57:00 | 3 | non_conforme |\n", + "| 062000735 | nitrate | dernier_prel | 2025-02-24 09:23:00 | 3 | conforme |\n", + "| 062000738 | nitrate | dernier_prel | 2025-02-12 10:08:00 | 3 | conforme |\n", + "| 062000769 | nitrate | dernier_prel | 2025-01-13 10:26:00 | 3 | conforme |\n", + "| 062000776 | nitrate | dernier_prel | 2024-11-25 08:22:00 | 3 | conforme |\n", + "| 062000830 | nitrate | dernier_prel | 2025-02-25 12:33:00 | 3 | conforme |\n", + "| 062000863 | nitrate | dernier_prel | 2025-01-07 08:40:00 | 3 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------+\n", "Truncated to displaylimit of 10." ] }, @@ -561,7 +561,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5059502bcb4f449e9f9fc1ec3cdd77e1", + "model_id": "ff52d6b6e6f843d2a28db5533585d651", "version_major": 2, "version_minor": 0 }, @@ -585,15 +585,15 @@ " \n", " \n", " donnee_manquante\n", - " 1162\n", + " 1169\n", " \n", " \n", - " non_conforme\n", - " 269\n", + " conforme\n", + " 20686\n", " \n", " \n", - " conforme\n", - " 20704\n", + " non_conforme\n", + " 269\n", " \n", " \n", "" @@ -602,9 +602,9 @@ "+------------------+--------------+\n", "| resultat | count_star() |\n", "+------------------+--------------+\n", - "| donnee_manquante | 1162 |\n", + "| donnee_manquante | 1169 |\n", + "| conforme | 20686 |\n", "| non_conforme | 269 |\n", - "| conforme | 20704 |\n", "+------------------+--------------+" ] }, @@ -643,7 +643,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a1847396cf544b84862cf51d3514e1a9", + "model_id": "a96537ab0b9f4f89bd913ccf11035bc0", "version_major": 2, "version_minor": 0 }, @@ -715,7 +715,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bcad3e0f29c643a494ac8920d4d1a5d1", + "model_id": "2f02427c65a2483982d14158533ba333", "version_major": 2, "version_minor": 0 }, @@ -739,9 +739,9 @@ " \n", " \n", " \n", - " non_conforme\n", - " 002000060\n", - " 095000346\n", + " donnee_manquante\n", + " 001000251\n", + " 976003881\n", " \n", " \n", " conforme\n", @@ -749,9 +749,9 @@ " 976003489\n", " \n", " \n", - " donnee_manquante\n", - " 001000251\n", - " 976003881\n", + " non_conforme\n", + " 002000060\n", + " 095000346\n", " \n", " \n", "" @@ -760,9 +760,9 @@ "+------------------+--------------+--------------+\n", "| resultat | min_cdreseau | max_cdreseau |\n", "+------------------+--------------+--------------+\n", - "| non_conforme | 002000060 | 095000346 |\n", - "| conforme | 001000003 | 976003489 |\n", "| donnee_manquante | 001000251 | 976003881 |\n", + "| conforme | 001000003 | 976003489 |\n", + "| non_conforme | 002000060 | 095000346 |\n", "+------------------+--------------+--------------+" ] }, @@ -809,7 +809,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ef9a50f69b794ae9862fec9d38d85ac7", + "model_id": "2f660f72adc44235bf8ad78007bab56c", "version_major": 2, "version_minor": 0 }, @@ -1147,7 +1147,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dcbe2dd72bd94fe4ad82115686a395eb", + "model_id": "f22ecc549857467ca4ad388576cc30b4", "version_major": 2, "version_minor": 0 }, @@ -1324,7 +1324,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "90c402d060c64f848bc8868c9b5ae212", + "model_id": "6810d3c6d92c4864964687c62b4287c9", "version_major": 2, "version_minor": 0 }, @@ -1665,7 +1665,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 2, "id": "dbdcec00-8e5c-4eb0-9fac-2a5f90709383", "metadata": {}, "outputs": [ @@ -1684,7 +1684,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ee1b9b581f6948db80e00512efe67a8f", + "model_id": "0bc8fea3bd484446ae765a59dfa149bb", "version_major": 2, "version_minor": 0 }, @@ -1711,47 +1711,47 @@ " \n", " \n", " \n", - " 001000241\n", + " 001000311\n", " nitrate\n", " dernier_prel\n", - " 2025-02-10 11:04:00\n", + " 2024-12-10 12:02:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000304\n", + " 001000336\n", " nitrate\n", " dernier_prel\n", - " 2025-02-11 10:10:00\n", + " 2024-10-31 10:45:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000313\n", + " 001000373\n", " nitrate\n", " dernier_prel\n", - " 2024-11-27 11:00:00\n", + " 2024-11-29 11:03:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000387\n", + " 001000386\n", " nitrate\n", " dernier_prel\n", - " 2025-02-13 10:48:00\n", + " 2024-11-27 08:19:00\n", " 3\n", - " donnee_manquante\n", + " conforme\n", " \n", " \n", - " 001000427\n", + " 001000388\n", " nitrate\n", " dernier_prel\n", - " 2025-01-24 11:15:00\n", + " 2024-06-05 08:47:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000464\n", + " 001000447\n", " nitrate\n", " dernier_prel\n", " 2025-01-16 09:17:00\n", @@ -1759,34 +1759,34 @@ " conforme\n", " \n", " \n", - " 001000490\n", + " 001000498\n", " nitrate\n", " dernier_prel\n", - " 2024-09-27 11:53:00\n", + " 2024-06-03 08:15:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000585\n", + " 001000525\n", " nitrate\n", " dernier_prel\n", - " 2025-02-21 09:15:00\n", + " 2025-02-03 10:21:00\n", " 3\n", " conforme\n", " \n", " \n", - " 001000824\n", + " 001000584\n", " nitrate\n", " dernier_prel\n", - " 2024-10-18 09:54:00\n", + " 2025-01-09 10:01:00\n", " 3\n", - " conforme\n", + " donnee_manquante\n", " \n", " \n", - " 001000865\n", + " 001000655\n", " nitrate\n", " dernier_prel\n", - " 2025-02-11 09:27:00\n", + " 2025-01-27 09:48:00\n", " 3\n", " conforme\n", " \n", @@ -1798,21 +1798,21 @@ "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000241 | nitrate | dernier_prel | 2025-02-10 11:04:00 | 3 | conforme |\n", - "| 001000304 | nitrate | dernier_prel | 2025-02-11 10:10:00 | 3 | conforme |\n", - "| 001000313 | nitrate | dernier_prel | 2024-11-27 11:00:00 | 3 | conforme |\n", - "| 001000387 | nitrate | dernier_prel | 2025-02-13 10:48:00 | 3 | donnee_manquante |\n", - "| 001000427 | nitrate | dernier_prel | 2025-01-24 11:15:00 | 3 | conforme |\n", - "| 001000464 | nitrate | dernier_prel | 2025-01-16 09:17:00 | 3 | conforme |\n", - "| 001000490 | nitrate | dernier_prel | 2024-09-27 11:53:00 | 3 | conforme |\n", - "| 001000585 | nitrate | dernier_prel | 2025-02-21 09:15:00 | 3 | conforme |\n", - "| 001000824 | nitrate | dernier_prel | 2024-10-18 09:54:00 | 3 | conforme |\n", - "| 001000865 | nitrate | dernier_prel | 2025-02-11 09:27:00 | 3 | conforme |\n", + "| 001000311 | nitrate | dernier_prel | 2024-12-10 12:02:00 | 3 | conforme |\n", + "| 001000336 | nitrate | dernier_prel | 2024-10-31 10:45:00 | 3 | conforme |\n", + "| 001000373 | nitrate | dernier_prel | 2024-11-29 11:03:00 | 3 | conforme |\n", + "| 001000386 | nitrate | dernier_prel | 2024-11-27 08:19:00 | 3 | conforme |\n", + "| 001000388 | nitrate | dernier_prel | 2024-06-05 08:47:00 | 3 | conforme |\n", + "| 001000447 | nitrate | dernier_prel | 2025-01-16 09:17:00 | 3 | conforme |\n", + "| 001000498 | nitrate | dernier_prel | 2024-06-03 08:15:00 | 3 | conforme |\n", + "| 001000525 | nitrate | dernier_prel | 2025-02-03 10:21:00 | 3 | conforme |\n", + "| 001000584 | nitrate | dernier_prel | 2025-01-09 10:01:00 | 3 | donnee_manquante |\n", + "| 001000655 | nitrate | dernier_prel | 2025-01-27 09:48:00 | 3 | conforme |\n", "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", "Truncated to displaylimit of 10." ] }, - "execution_count": 11, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -2032,7 +2032,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "04a10f29f2204ff38289c79a1b04b10c", + "model_id": "03a735b239374693b51eb407bbfba319", "version_major": 2, "version_minor": 0 }, @@ -2055,16 +2055,16 @@ " \n", " \n", " \n", - " non_conforme\n", - " 269\n", + " conforme\n", + " 17329\n", " \n", " \n", - " conforme\n", - " 17338\n", + " non_conforme\n", + " 269\n", " \n", " \n", " donnee_manquante\n", - " 4528\n", + " 4526\n", " \n", " \n", "" @@ -2073,9 +2073,9 @@ "+------------------+--------------+\n", "| resultat | count_star() |\n", "+------------------+--------------+\n", + "| conforme | 17329 |\n", "| non_conforme | 269 |\n", - "| conforme | 17338 |\n", - "| donnee_manquante | 4528 |\n", + "| donnee_manquante | 4526 |\n", "+------------------+--------------+" ] }, @@ -2114,7 +2114,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6db26e98b8ee494aa91c4190644dcd44", + "model_id": "e870dbf0ff544b178cb7231b4538d23e", "version_major": 2, "version_minor": 0 }, @@ -2186,7 +2186,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9f02b918c1054b138d148edd1a0ebc21", + "model_id": "96da167393374630b0e28e87738828b0", "version_major": 2, "version_minor": 0 }, @@ -2221,14 +2221,6 @@ " conforme\n", " \n", " \n", - " 001000258\n", - " nitrate\n", - " dernier_prel\n", - " 2025-01-21 09:45:00\n", - " 3\n", - " donnee_manquante\n", - " \n", - " \n", " 001000260\n", " nitrate\n", " dernier_prel\n", @@ -2253,22 +2245,145 @@ " conforme\n", " \n", " \n", - " 001000387\n", + " 002000060\n", " nitrate\n", " dernier_prel\n", - " 2025-02-13 10:48:00\n", + " 2024-10-18 08:50:00\n", " 3\n", + " non_conforme\n", + " \n", + " \n", + " 073002059\n", + " nitrate\n", + " dernier_prel\n", + " 2024-10-23 09:28:00\n", + " 1\n", " donnee_manquante\n", " \n", " \n", - " 002000060\n", + " 088002296\n", " nitrate\n", " dernier_prel\n", - " 2024-10-18 08:50:00\n", + " 2024-09-27 12:00:00\n", + " 1\n", + " donnee_manquante\n", + " \n", + " \n", + " 095000346\n", + " nitrate\n", + " dernier_prel\n", + " 2025-02-27 08:35:00\n", " 3\n", " non_conforme\n", " \n", " \n", + " 973000028\n", + " nitrate\n", + " dernier_prel\n", + " 2024-12-02 11:30:00\n", + " 2\n", + " conforme\n", + " \n", + " \n", + "" + ], + "text/plain": [ + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", + "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | conforme |\n", + "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", + "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | conforme |\n", + "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | conforme |\n", + "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", + "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", + "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | donnee_manquante |\n", + "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | non_conforme |\n", + "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", + " SELECT \n", + " *\n", + " FROM\n", + " int__resultats_nitrites_udi_dernier_dateCheck\n", + "WHERE\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", + "ORDER BY \n", + " cdreseau" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "984e0235-3790-4da7-bb47-50e23e7fa61b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c1bc7e8aea3b422b88d2e462c8dd8ea4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -2285,36 +2400,29 @@ " \n", " \n", " \n", - " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000258nitratedernier_prel2025-01-21 09:45:003donnee_manquante
001000387nitratedernier_prel2025-02-13 10:48:003donnee_manquante
008000546nitratedernier_prelnon_conforme
073002059095000346nitratedernier_prel2024-10-23 09:28:001donnee_manquante2025-02-27 08:35:003non_conforme
\n", - "Truncated to displaylimit of 10." + "" ], "text/plain": [ "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | conforme |\n", "| 001000258 | nitrate | dernier_prel | 2025-01-21 09:45:00 | 3 | donnee_manquante |\n", - "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", - "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | conforme |\n", - "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | conforme |\n", "| 001000387 | nitrate | dernier_prel | 2025-02-13 10:48:00 | 3 | donnee_manquante |\n", - "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", "| 008000546 | nitrate | dernier_prel | 2025-02-07 10:38:00 | 2 | non_conforme |\n", "| 008000681 | nitrate | dernier_prel | 2025-02-12 11:38:00 | 2 | non_conforme |\n", - "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "Truncated to displaylimit of 10." + "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | non_conforme |\n", + "+-----------+-----------+--------------+-----------------------+---------------+------------------+" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -2326,15 +2434,14 @@ " FROM\n", " int__resultats_nitrites_udi_dernier_dateCheck\n", "WHERE\n", - " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293',\n", - " '001000258','001000387','008000681','008000546')\n", + " cdreseau IN('001000258','001000387','008000681','008000546','095000346')\n", "ORDER BY \n", " cdreseau" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 5, "id": "4033d129-eeab-4ec9-ac40-39a4101e3083", "metadata": {}, "outputs": [ @@ -2353,7 +2460,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7e1709e2f4e34e068d5a3fe1ea29ce6d", + "model_id": "ad9a179083cb44d38acae7d24f0b887f", "version_major": 2, "version_minor": 0 }, @@ -2370,9 +2477,11 @@ "\n", " \n", " \n", - " \n", " \n", " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -2381,31 +2490,29 @@ "
periodecdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
" ], "text/plain": [ - "+---------+----------+-----------+----------+\n", - "| periode | cdreseau | categorie | resultat |\n", - "+---------+----------+-----------+----------+\n", - "+---------+----------+-----------+----------+" + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "+----------+-----------+---------+-----------------------+---------------+----------+" ] }, - "execution_count": 15, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", + "-- dernier relevé\n", "SELECT\n", - " 'dernier relevé' AS periode,\n", - " cdreseau,\n", - " categorie,\n", - " resultat,\n", + "*\n", "FROM\n", " int__resultats_nitrites_udi_dernier_dateCheck\n", "WHERE\n", " (\n", " cdreseau = '001000003'\n", " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-12-26 09:08:00'\n", + " AND dernier_prel_datetime = '2025-01-21 12:35:00'\n", " AND resultat != 'conforme'\n", " )\n", " OR\n", @@ -2419,7 +2526,7 @@ " (\n", " cdreseau = '001000268'\n", " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-11-28 08:25:00'\n", + " AND dernier_prel_datetime = '2024-12-06 12:02:00'\n", " AND resultat != 'conforme'\n", " )\n", " OR\n", @@ -2435,7 +2542,6 @@ " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2024-10-23 09:28:00'\n", " AND resultat != 'donnee_manquante'\n", - " AND nb_parametres = 1\n", " )\n", " OR\n", " (\n", @@ -2443,22 +2549,20 @@ " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2024-09-27 12:00:00'\n", " AND resultat != 'donnee_manquante'\n", - " AND nb_parametres = 1\n", " )\n", " OR\n", " (\n", " cdreseau = '095000346'\n", " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-12-17 08:55:00'\n", + " AND dernier_prel_datetime = '2025-02-27 08:35:00'\n", " AND resultat != 'non_conforme'\n", " )\n", " OR\n", " (\n", " cdreseau = '973000028'\n", " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-12-17 08:55:00'\n", + " AND dernier_prel_datetime = '2024-12-02 11:30:00'\n", " AND resultat != 'conforme'\n", - " AND nb_parametres = 2\n", " )\n", " OR\n", " (\n", @@ -2466,15 +2570,13 @@ " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-01-21 09:45:00'\n", " AND resultat != 'donnee_manquante'\n", - " AND nb_parametres = 2\n", " )\n", " OR\n", " (\n", " cdreseau = '001000387'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-02-13 10:48:00'\n", - " AND resultat != 'conforme'\n", - " AND nb_parametres = 2\n", + " AND resultat != 'donnee_manquante'\n", " )\n", " OR\n", " (\n", @@ -2482,7 +2584,6 @@ " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-02-12 11:38:00'\n", " AND resultat != 'non_conforme'\n", - " AND nb_parametres = 2\n", " )\n", " OR\n", " (\n", @@ -2490,14 +2591,13 @@ " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-02-07 10:38:00'\n", " AND resultat != 'non_conforme'\n", - " AND nb_parametres = 2\n", - " )" + " )\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "e5c04a09-1064-4041-8d2a-3d662202801a", + "id": "35aaf65c-8ba7-437e-bcbf-5fbed491bc39", "metadata": {}, "outputs": [], "source": [] diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 1b24efd9..a72ce88a 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -13,7 +13,7 @@ WHERE ( cdreseau = '001000003' AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-12-26 09:08:00' + AND dernier_prel_datetime = '2025-01-21 12:35:00' AND resultat != 'conforme' ) OR @@ -27,7 +27,7 @@ WHERE ( cdreseau = '001000268' AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-11-28 08:25:00' + AND dernier_prel_datetime = '2024-12-06 12:02:00' AND resultat != 'conforme' ) OR @@ -43,7 +43,6 @@ WHERE AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-10-23 09:28:00' AND resultat != 'donnee_manquante' - AND nb_parametres = 1 ) OR ( @@ -51,22 +50,20 @@ WHERE AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-09-27 12:00:00' AND resultat != 'donnee_manquante' - AND nb_parametres = 1 ) OR ( cdreseau = '095000346' AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-12-17 08:55:00' + AND dernier_prel_datetime = '2025-02-27 08:35:00' AND resultat != 'non_conforme' ) OR ( cdreseau = '973000028' AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-12-17 08:55:00' + AND dernier_prel_datetime = '2024-12-02 11:30:00' AND resultat != 'conforme' - AND nb_parametres = 2 ) OR ( @@ -74,15 +71,13 @@ WHERE AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-01-21 09:45:00' AND resultat != 'donnee_manquante' - AND nb_parametres = 2 ) OR ( cdreseau = '001000387' AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-02-13 10:48:00' - AND resultat != 'conforme' - AND nb_parametres = 2 + AND resultat != 'donnee_manquante' ) OR ( @@ -90,7 +85,6 @@ WHERE AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-02-12 11:38:00' AND resultat != 'non_conforme' - AND nb_parametres = 2 ) OR ( @@ -98,5 +92,4 @@ WHERE AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-02-07 10:38:00' AND resultat != 'non_conforme' - AND nb_parametres = 2 ) From 7af80c1c55312133f7eeeb26b4b00f498ceb59cd Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Tue, 15 Apr 2025 16:15:06 +0200 Subject: [PATCH 18/30] feat: update method for nitrates/nitrites + add missing field into yaml --- .../Test_depassements_no2vsno3.ipynb | 360 +++ analytics/notebooks/resultats_nitrates.ipynb | 2232 +++++------------ dbt_/models/intermediate/_int__models.yml | 4 + .../intermediate/cvm/_int__cvm_models.yaml | 27 + .../int__resultats_cvm_commune_dernier.sql | 2 +- .../cvm/int__resultats_cvm_udi_dernier.sql | 2 +- .../nitrites/_int__nitrites_models.yaml | 14 +- .../int__resultats_nitrites_udi_dernier.sql | 146 +- dbt_/tests/test_nitrates_results.sql | 46 +- 9 files changed, 1148 insertions(+), 1685 deletions(-) create mode 100644 analytics/notebooks/Test_depassements_no2vsno3.ipynb diff --git a/analytics/notebooks/Test_depassements_no2vsno3.ipynb b/analytics/notebooks/Test_depassements_no2vsno3.ipynb new file mode 100644 index 00000000..82e4bfa0 --- /dev/null +++ b/analytics/notebooks/Test_depassements_no2vsno3.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "id": "443a3c28-b272-4788-9995-0bcadc959d15", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The sql extension is already loaded. To reload it, use:\n", + " %reload_ext sql\n" + ] + } + ], + "source": [ + "%load_ext sql\n", + "%sql duckdb:///../../database/data.duckdb\n", + "%config SqlMagic.named_parameters=\"enabled\" " + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "33985e98-d332-40dc-9038-4bd998e5567d", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6b6b6d2428cc474fadf32706ae5d5f72", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
anneenb_prelevementsnb_dep_3critèrespercent_3critèresnb_prelevements_n02nb_dep_n02percent_n02nb_prelevements_n03nb_dep_n03percent_n03
202012938617451.0129385270.012938117031.0
2025181382851.01813810.0176812831.0
202112649318231.0126487170.012649018021.0
202212874516051.0128736160.012873715861.0
202412773419801.0127733160.012773319601.0
202312917214471.0129171350.012915414051.0
" + ], + "text/plain": [ + "+-------+-----------------+------------------+-------------------+---------------------+------------+-------------+---------------------+------------+-------------+\n", + "| annee | nb_prelevements | nb_dep_3critères | percent_3critères | nb_prelevements_n02 | nb_dep_n02 | percent_n02 | nb_prelevements_n03 | nb_dep_n03 | percent_n03 |\n", + "+-------+-----------------+------------------+-------------------+---------------------+------------+-------------+---------------------+------------+-------------+\n", + "| 2020 | 129386 | 1745 | 1.0 | 129385 | 27 | 0.0 | 129381 | 1703 | 1.0 |\n", + "| 2025 | 18138 | 285 | 1.0 | 18138 | 1 | 0.0 | 17681 | 283 | 1.0 |\n", + "| 2021 | 126493 | 1823 | 1.0 | 126487 | 17 | 0.0 | 126490 | 1802 | 1.0 |\n", + "| 2022 | 128745 | 1605 | 1.0 | 128736 | 16 | 0.0 | 128737 | 1586 | 1.0 |\n", + "| 2024 | 127734 | 1980 | 1.0 | 127733 | 16 | 0.0 | 127733 | 1960 | 1.0 |\n", + "| 2023 | 129172 | 1447 | 1.0 | 129171 | 35 | 0.0 | 129154 | 1405 | 1.0 |\n", + "+-------+-----------------+------------------+-------------------+---------------------+------------+-------------+---------------------+------------+-------------+" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "WITH\n", + "prels AS (\n", + " SELECT\n", + " de_partition AS annee,\n", + " cdreseau,\n", + " referenceprel,\n", + " datetimeprel,\n", + " COUNT(DISTINCT cdparametresiseeaux) AS nb_parametres,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN valtraduite\n", + " ELSE 0\n", + " END\n", + " ) AS valtraduite_no3_no2\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrate'\n", + " GROUP BY\n", + " annee,\n", + " cdreseau,\n", + " referenceprel,\n", + " datetimeprel,\n", + "),\n", + "\n", + "valeur_ref AS (\n", + " SELECT\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO3_NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no3_no2,\n", + " MAX(\n", + " CASE\n", + " WHEN\n", + " cdparametresiseeaux = 'NO2'\n", + " THEN limite_qualite\n", + " END\n", + " ) AS limite_qualite_no2\n", + " FROM\n", + " int__valeurs_de_reference\n", + " WHERE\n", + " categorie_1 = 'nitrate'\n", + ")\n", + "\n", + "SELECT\n", + " -- prels.cdreseau,\n", + " prels.annee,\n", + " -- 'nitrate' AS categorie,\n", + " -- 'bilan_annuel_' || prels.annee AS periode,\n", + " COUNT(DISTINCT prels.referenceprel) AS nb_prelevements,\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n", + " OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n", + " OR prels.valtraduite_no3_no2\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " THEN prels.referenceprel\n", + " END\n", + " ) AS nb_dep_3critères,\n", + " trunc((\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n", + " OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n", + " OR prels.valtraduite_no3_no2\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3\n", + " >= valeur_ref.limite_qualite_no3_no2\n", + " THEN prels.referenceprel\n", + " END\n", + " )::float\n", + " /\n", + " COUNT(DISTINCT prels.referenceprel)::float\n", + " )*100) AS percent_3critères,\n", + "\n", + " COUNT(DISTINCT CASE WHEN valtraduite_no2 IS NOT NULL THEN prels.referenceprel END) AS nb_prelevements_n02,\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n", + " THEN prels.referenceprel\n", + " END\n", + " ) AS nb_dep_n02,\n", + " trunc((\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2\n", + " THEN prels.referenceprel\n", + " END\n", + " )::float\n", + " /\n", + " COUNT(DISTINCT CASE WHEN valtraduite_no2 IS NOT NULL THEN prels.referenceprel END)::float\n", + " )*100) AS percent_n02,\n", + "\n", + " COUNT(DISTINCT CASE WHEN valtraduite_no3 IS NOT NULL THEN prels.referenceprel END) AS nb_prelevements_n03,\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n", + " THEN prels.referenceprel\n", + " END\n", + " ) AS nb_dep_n03,\n", + " trunc((\n", + " COUNT(\n", + " DISTINCT\n", + " CASE\n", + " WHEN\n", + " prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3\n", + " THEN prels.referenceprel\n", + " END\n", + " )::float\n", + " /\n", + " COUNT(DISTINCT CASE WHEN valtraduite_no2 IS NOT NULL THEN prels.referenceprel END)::float\n", + " )*100) AS percent_n03\n", + "\n", + "FROM prels\n", + "CROSS JOIN valeur_ref\n", + "GROUP BY /*prels.cdreseau,*/ prels.annee\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2b49cfb-e586-4523-991e-41a25a7d7b02", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/analytics/notebooks/resultats_nitrates.ipynb b/analytics/notebooks/resultats_nitrates.ipynb index 836b8eb9..4f414145 100644 --- a/analytics/notebooks/resultats_nitrates.ipynb +++ b/analytics/notebooks/resultats_nitrates.ipynb @@ -77,11 +77,11 @@ " \n", " \n", " \n", - " NO2\n", + " NO3\n", " nitrate\n", " None\n", " None\n", - " 0.5\n", + " 50.0\n", " mg/L\n", " None\n", " None\n", @@ -90,11 +90,11 @@ " 1\n", " \n", " \n", - " NO3\n", + " NO2\n", " nitrate\n", " None\n", " None\n", - " 50.0\n", + " 0.5\n", " mg/L\n", " None\n", " None\n", @@ -122,8 +122,8 @@ "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+\n", "| cdparametresiseeaux | categorie_1 | categorie_2 | categorie_3 | limite_qualite | limite_qualite_unite | valeur_sanitaire_1 | valeur_sanitaire_1_unite | valeur_sanitaire_2 | valeur_sanitaire_2_unite | nb_rows |\n", "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+\n", - "| NO2 | nitrate | None | None | 0.5 | mg/L | None | None | None | None | 1 |\n", "| NO3 | nitrate | None | None | 50.0 | mg/L | None | None | None | None | 1 |\n", + "| NO2 | nitrate | None | None | 0.5 | mg/L | None | None | None | None | 1 |\n", "| NO3_NO2 | nitrate | None | None | 1.0 | None | None | None | None | None | 1 |\n", "+---------------------+-------------+-------------+-------------+----------------+----------------------+--------------------+--------------------------+--------------------+--------------------------+---------+" ] @@ -225,18 +225,17 @@ "metadata": {}, "source": [ "**Les règles choisies**\n", - " \n", - "_Si on a les 3 paramètres disponibles_\n", - "- Si Nitrates < 50 mg/L et nitrites < 0,5 mg/L et nitrate/50 + nitrites/3 < 1 mg/L --> CONFORME\n", - "- Si Nitrates >= 50 mg/L et/ou nitrites >= 0,5 mg/L et/ou nitrate/50 + nitrites/3 >= 1 mg/L --> NON CONFORME\n", - "\n", "\n", - "_Si on n'a pas nitrate/50 + nitrites/3 mais qu'on a nitrate et nitrites_
\n", - "==> on le calcule et on applique comme ci-dessus\n", + "Pour le choix du dernier prélèvement (date et id) : on se réfère uniquement à nitrate (no3) \n", + "Mais on doit regarder ensuite toutes les données (no3/no2) des prélèvements sélectionnés\n", "\n", - "- Si on n'a pas nitrate --> donnee_manquante\n", - "- Si on n'a pas nitrites et que nitrate >= 50 mg/L --> NON CONFORME\n", - "- Si on n'a pas nitrites et que nitrate < 50 mg/L --> donnee_manquant \n" + "- Si on n'a pas nitrate (no3) : pas de données dans la table. ca sera \"pas recherché\" sur le site\n", + "- Si nitrates (no3) ET nitrites (no2) \n", + " - Si Nitrates < 50 mg/L et nitrites < 0,5 mg/L et nitrate/50 + nitrites/3 < 1 mg/L --> inf_limite_qualite\n", + " - Si Nitrates >= 50 mg/L et/ou nitrites >= 0,5 mg/L et/ou nitrate/50 + nitrites/3 >= 1 mg/L --> sup_limite_qualite\n", + "- Si nitrates (no3) et pas nitrites (no2) : \n", + " - Nitrates < 50 mg/L --> inf_limite_qualite \n", + " - Si Nitrates >= 50 mg/L -> sup_limite_qualite" ] }, { @@ -260,7 +259,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c7b7f35ae9004e30bdf76cb212cb50ee", + "model_id": "3066bdabe1a64b02991ce29b03368a80", "version_major": 2, "version_minor": 0 }, @@ -287,104 +286,104 @@ " \n", " \n", " \n", - " 062000617\n", + " 043000382\n", " nitrate\n", " dernier_prel\n", - " 2025-02-07 11:19:00\n", + " 2024-08-21 11:04:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000620\n", + " 048000421\n", " nitrate\n", " dernier_prel\n", - " 2024-09-25 11:59:00\n", + " 2024-05-23 12:04:00\n", " 2\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000655\n", + " 055000098\n", " nitrate\n", " dernier_prel\n", - " 2024-10-03 11:16:00\n", + " 2024-08-05 08:48:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000660\n", + " 066000236\n", " nitrate\n", " dernier_prel\n", - " 2025-02-13 08:57:00\n", - " 3\n", - " non_conforme\n", + " 2024-06-19 10:33:00\n", + " 2\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000735\n", + " 066000312\n", " nitrate\n", " dernier_prel\n", - " 2025-02-24 09:23:00\n", - " 3\n", - " conforme\n", + " 2024-10-29 10:32:00\n", + " 2\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000738\n", + " 067001856\n", " nitrate\n", " dernier_prel\n", - " 2025-02-12 10:08:00\n", + " 2024-12-06 08:15:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000769\n", + " 068001002\n", " nitrate\n", " dernier_prel\n", - " 2025-01-13 10:26:00\n", - " 3\n", - " conforme\n", + " 2024-10-28 09:40:00\n", + " 2\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000776\n", + " 073001318\n", " nitrate\n", " dernier_prel\n", - " 2024-11-25 08:22:00\n", + " 2024-12-19 09:35:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000830\n", + " 073001176\n", " nitrate\n", " dernier_prel\n", - " 2025-02-25 12:33:00\n", + " 2024-08-28 11:59:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", - " 062000863\n", + " 080000913\n", " nitrate\n", " dernier_prel\n", - " 2025-01-07 08:40:00\n", + " 2024-11-28 14:38:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", "\n", "Truncated to displaylimit of 10." ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+--------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+--------------+\n", - "| 062000617 | nitrate | dernier_prel | 2025-02-07 11:19:00 | 3 | conforme |\n", - "| 062000620 | nitrate | dernier_prel | 2024-09-25 11:59:00 | 2 | conforme |\n", - "| 062000655 | nitrate | dernier_prel | 2024-10-03 11:16:00 | 3 | conforme |\n", - "| 062000660 | nitrate | dernier_prel | 2025-02-13 08:57:00 | 3 | non_conforme |\n", - "| 062000735 | nitrate | dernier_prel | 2025-02-24 09:23:00 | 3 | conforme |\n", - "| 062000738 | nitrate | dernier_prel | 2025-02-12 10:08:00 | 3 | conforme |\n", - "| 062000769 | nitrate | dernier_prel | 2025-01-13 10:26:00 | 3 | conforme |\n", - "| 062000776 | nitrate | dernier_prel | 2024-11-25 08:22:00 | 3 | conforme |\n", - "| 062000830 | nitrate | dernier_prel | 2025-02-25 12:33:00 | 3 | conforme |\n", - "| 062000863 | nitrate | dernier_prel | 2025-01-07 08:40:00 | 3 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+--------------+\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------------+\n", + "| 043000382 | nitrate | dernier_prel | 2024-08-21 11:04:00 | 3 | inf_limite_qualite |\n", + "| 048000421 | nitrate | dernier_prel | 2024-05-23 12:04:00 | 2 | inf_limite_qualite |\n", + "| 055000098 | nitrate | dernier_prel | 2024-08-05 08:48:00 | 3 | inf_limite_qualite |\n", + "| 066000236 | nitrate | dernier_prel | 2024-06-19 10:33:00 | 2 | inf_limite_qualite |\n", + "| 066000312 | nitrate | dernier_prel | 2024-10-29 10:32:00 | 2 | inf_limite_qualite |\n", + "| 067001856 | nitrate | dernier_prel | 2024-12-06 08:15:00 | 3 | inf_limite_qualite |\n", + "| 068001002 | nitrate | dernier_prel | 2024-10-28 09:40:00 | 2 | inf_limite_qualite |\n", + "| 073001318 | nitrate | dernier_prel | 2024-12-19 09:35:00 | 3 | inf_limite_qualite |\n", + "| 073001176 | nitrate | dernier_prel | 2024-08-28 11:59:00 | 3 | inf_limite_qualite |\n", + "| 080000913 | nitrate | dernier_prel | 2024-11-28 14:38:00 | 3 | inf_limite_qualite |\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------------+\n", "Truncated to displaylimit of 10." ] }, @@ -399,11 +398,8 @@ "last_pvl AS (\n", " SELECT\n", " cdreseau,\n", - " categorie,\n", - " cdparametresiseeaux,\n", - " limite_qualite,\n", + " referenceprel,\n", " datetimeprel,\n", - " valtraduite,\n", " ROW_NUMBER()\n", " OVER (\n", " PARTITION BY cdreseau, cdparametresiseeaux\n", @@ -411,12 +407,13 @@ " )\n", " AS row_number\n", " FROM\n", - " int__resultats_udi_communes\n", + " int__resultats_udi_communes\n", " WHERE\n", " categorie = 'nitrate'\n", - " AND\n", - " -- On garde les prélèvements de moins d'un an\n", + " AND cdparametresiseeaux = 'NO3'\n", + " AND -- On garde les prélèvements de moins d'un an\n", " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + " AND valtraduite IS NOT NULL \n", "),\n", "\n", "valeur_ref AS (\n", @@ -450,51 +447,65 @@ "\n", "split_nitrites AS (\n", " SELECT\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie,\n", - " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", - " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", + " resultats.cdreseau,\n", + " resultats.categorie,\n", + " resultats.datetimeprel AS dernier_prel_datetime,\n", + " COUNT(DISTINCT resultats.cdparametresiseeaux) AS nb_parametres,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.valtraduite\n", + " resultats.cdparametresiseeaux = 'NO3'\n", + " THEN resultats.valtraduite\n", " END\n", " ) AS valtraduite_no3,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.valtraduite\n", + " resultats.cdparametresiseeaux = 'NO2'\n", + " THEN resultats.valtraduite\n", " END\n", " ) AS valtraduite_no2,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", - " THEN last_pvl.valtraduite\n", + " resultats.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN resultats.valtraduite\n", " END\n", " ) AS valtraduite_no3_no2\n", " FROM\n", " last_pvl\n", + " LEFT JOIN\n", + " int__resultats_udi_communes AS resultats\n", + " ON\n", + " resultats.cdreseau = last_pvl.cdreseau\n", + " AND resultats.referenceprel = last_pvl.referenceprel\n", " WHERE\n", - " last_pvl.row_number = 1\n", + " last_pvl.row_number = 1 \n", + " -- on ne veut que le dernier prélèvement\n", + " AND resultats.categorie = 'nitrate' \n", + " -- on ne veut que les résultats associé aux nitrates/nitrites\n", " GROUP BY\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie\n", + " resultats.cdreseau,\n", + " resultats.categorie,\n", + " resultats.datetimeprel\n", "),\n", "\n", "split_nitrites_with_ref AS (\n", " SELECT\n", - " split_nitrites.*,\n", + " split_nitrites.cdreseau,\n", + " split_nitrites.categorie,\n", + " split_nitrites.nb_parametres,\n", + " split_nitrites.dernier_prel_datetime,\n", " valeur_ref.limite_qualite_no3,\n", - " valeur_ref.limite_qualite_no3_no2,\n", " valeur_ref.limite_qualite_no2,\n", - " split_nitrites.valtraduite_no3 / 50\n", - " + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc\n", - " FROM split_nitrites\n", - " CROSS JOIN valeur_ref\n", - "\n", + " valeur_ref.limite_qualite_no3_no2,\n", + " split_nitrites.valtraduite_no3_no2,\n", + " split_nitrites.valtraduite_no3,\n", + " split_nitrites.valtraduite_no2\n", + " FROM\n", + " split_nitrites\n", + " CROSS JOIN\n", + " valeur_ref\n", ")\n", "\n", "SELECT\n", @@ -504,37 +515,55 @@ " split_nitrites_with_ref.dernier_prel_datetime,\n", " split_nitrites_with_ref.nb_parametres,\n", " CASE\n", + " WHEN -- Si nitrates (no3) et pas nitrites (no2)\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL \n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NULL\n", + " AND\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", + " THEN 'inf_limite_qualite'\n", " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL \n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NULL\n", + " AND\n", " split_nitrites_with_ref.valtraduite_no3\n", + " >= split_nitrites_with_ref.limite_qualite_no3\n", + " THEN 'sup_limite_qualite'\n", + " WHEN -- Si nitrates (no3) ET nitrites (no2)\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL\n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL\n", + " AND (\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " >= split_nitrites_with_ref.limite_qualite_no3\n", + " OR split_nitrites_with_ref.valtraduite_no2\n", + " >= split_nitrites_with_ref.limite_qualite_no2\n", + " OR COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " (\n", + " split_nitrites_with_ref.valtraduite_no3 / 50\n", + " + split_nitrites_with_ref.valtraduite_no2 / 3\n", + " )\n", + " )\n", + " >= split_nitrites_with_ref.limite_qualite_no3_no2\n", + " )\n", + " THEN 'sup_limite_qualite'\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL\n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL\n", + " AND split_nitrites_with_ref.valtraduite_no3\n", " < split_nitrites_with_ref.limite_qualite_no3\n", " AND split_nitrites_with_ref.valtraduite_no2\n", " < split_nitrites_with_ref.limite_qualite_no2\n", " AND COALESCE(\n", " split_nitrites_with_ref.valtraduite_no3_no2,\n", - " split_nitrites_with_ref.valtraduite_no3_no2_calc\n", + " (\n", + " split_nitrites_with_ref.valtraduite_no3 / 50\n", + " + split_nitrites_with_ref.valtraduite_no2 / 3\n", + " )\n", " )\n", " < split_nitrites_with_ref.limite_qualite_no3_no2\n", - " THEN 'conforme'\n", - " WHEN\n", - " split_nitrites_with_ref.valtraduite_no3\n", - " >= split_nitrites_with_ref.limite_qualite_no3\n", - " OR split_nitrites_with_ref.valtraduite_no2\n", - " >= split_nitrites_with_ref.limite_qualite_no2\n", - " OR COALESCE(\n", - " split_nitrites_with_ref.valtraduite_no3_no2,\n", - " split_nitrites_with_ref.valtraduite_no3_no2_calc\n", - " )\n", - " >= split_nitrites_with_ref.limite_qualite_no3_no2\n", - " THEN 'non_conforme'\n", - " WHEN\n", - " split_nitrites_with_ref.valtraduite_no3 IS NULL\n", - " OR (\n", - " split_nitrites_with_ref.valtraduite_no2 IS NULL\n", - " AND split_nitrites_with_ref.valtraduite_no3\n", - " < split_nitrites_with_ref.limite_qualite_no3\n", - " )\n", - " THEN 'donnee_manquante'\n", - " ELSE 'error'\n", + " THEN 'inf_limite_qualite'\n", + " ELSE 'erreur'\n", " END AS resultat\n", "FROM\n", " split_nitrites_with_ref\n" @@ -561,7 +590,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ff52d6b6e6f843d2a28db5533585d651", + "model_id": "9765c37b8ba34e33842642772307caed", "version_major": 2, "version_minor": 0 }, @@ -584,28 +613,23 @@ " \n", " \n", " \n", - " donnee_manquante\n", - " 1169\n", + " inf_limite_qualite\n", + " 21184\n", " \n", " \n", - " conforme\n", - " 20686\n", - " \n", - " \n", - " non_conforme\n", - " 269\n", + " sup_limite_qualite\n", + " 224\n", " \n", " \n", "" ], "text/plain": [ - "+------------------+--------------+\n", - "| resultat | count_star() |\n", - "+------------------+--------------+\n", - "| donnee_manquante | 1169 |\n", - "| conforme | 20686 |\n", - "| non_conforme | 269 |\n", - "+------------------+--------------+" + "+--------------------+--------------+\n", + "| resultat | count_star() |\n", + "+--------------------+--------------+\n", + "| inf_limite_qualite | 21184 |\n", + "| sup_limite_qualite | 224 |\n", + "+--------------------+--------------+" ] }, "execution_count": 5, @@ -626,7 +650,9 @@ "cell_type": "code", "execution_count": 6, "id": "2a287b57-fa15-4939-8209-7dcdf0e75eda", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "data": { @@ -643,7 +669,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a96537ab0b9f4f89bd913ccf11035bc0", + "model_id": "f48e5a03122345c3828600482cba31db", "version_major": 2, "version_minor": 0 }, @@ -691,12 +717,282 @@ "FROM \n", " int__resultats_nitrites_udi_dernier\n", "WHERE \n", - " resultat='error'" + " resultat='erreur'" + ] + }, + { + "cell_type": "markdown", + "id": "5cefcf3f-208f-4427-9e77-9155a20b7c65", + "metadata": {}, + "source": [ + "#### Cf cas cdreseau IN( '034005906') , referenceprel= 03400327764" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "ee06ff28-628c-481c-b9d9-eb8e23bba6b5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ], + "text/plain": [ + "Running query in 'duckdb:///../../database/data.duckdb'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c4298322f3e6460c8dc50d5e0e5f420f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
referenceprelcdparametresiseeauxde_partitionvaltraduitecategoriecategorie_2categorie_3limite_qualitevaleur_sanitaire_1valeur_sanitaire_2cdreseauinseecommunedatetimeprelrow_number
03400327273NO32025NonenitrateNoneNone50.0NoneNone034005906340882025-01-31 09:34:009
03400327273NO32025NonenitrateNoneNone50.0NoneNone034005906340872025-01-31 09:34:0010
03400327275NO32025NonenitrateNoneNone50.0NoneNone034005906340882025-01-28 07:50:0011
03400327275NO32025NonenitrateNoneNone50.0NoneNone034005906340872025-01-28 07:50:0012
03400327272NO32025NonenitrateNoneNone50.0NoneNone034005906340882025-01-08 10:17:0013
03400327272NO32025NonenitrateNoneNone50.0NoneNone034005906340872025-01-08 10:17:0014
03400316443NO320243.000nitrateNoneNone50.0NoneNone034005906340872024-12-03 09:23:0015
03400316443NO320243.000nitrateNoneNone50.0NoneNone034005906340882024-12-03 09:23:0016
03400327764NO32025NonenitrateNoneNone50.0NoneNone034005906340882025-02-21 10:16:001
03400327764NO32025NonenitrateNoneNone50.0NoneNone034005906340872025-02-21 10:16:002
\n", + "Truncated to displaylimit of 10." + ], + "text/plain": [ + "+---------------+---------------------+--------------+-------------+-----------+-------------+-------------+----------------+--------------------+--------------------+-----------+--------------+---------------------+------------+\n", + "| referenceprel | cdparametresiseeaux | de_partition | valtraduite | categorie | categorie_2 | categorie_3 | limite_qualite | valeur_sanitaire_1 | valeur_sanitaire_2 | cdreseau | inseecommune | datetimeprel | row_number |\n", + "+---------------+---------------------+--------------+-------------+-----------+-------------+-------------+----------------+--------------------+--------------------+-----------+--------------+---------------------+------------+\n", + "| 03400327273 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34088 | 2025-01-31 09:34:00 | 9 |\n", + "| 03400327273 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34087 | 2025-01-31 09:34:00 | 10 |\n", + "| 03400327275 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34088 | 2025-01-28 07:50:00 | 11 |\n", + "| 03400327275 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34087 | 2025-01-28 07:50:00 | 12 |\n", + "| 03400327272 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34088 | 2025-01-08 10:17:00 | 13 |\n", + "| 03400327272 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34087 | 2025-01-08 10:17:00 | 14 |\n", + "| 03400316443 | NO3 | 2024 | 3.000 | nitrate | None | None | 50.0 | None | None | 034005906 | 34087 | 2024-12-03 09:23:00 | 15 |\n", + "| 03400316443 | NO3 | 2024 | 3.000 | nitrate | None | None | 50.0 | None | None | 034005906 | 34088 | 2024-12-03 09:23:00 | 16 |\n", + "| 03400327764 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34088 | 2025-02-21 10:16:00 | 1 |\n", + "| 03400327764 | NO3 | 2025 | None | nitrate | None | None | 50.0 | None | None | 034005906 | 34087 | 2025-02-21 10:16:00 | 2 |\n", + "+---------------+---------------------+--------------+-------------+-----------+-------------+-------------+----------------+--------------------+--------------------+-----------+--------------+---------------------+------------+\n", + "Truncated to displaylimit of 10." + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + " SELECT\n", + "*,\n", + " ROW_NUMBER()\n", + " OVER (\n", + " PARTITION BY cdreseau, cdparametresiseeaux\n", + " ORDER BY datetimeprel DESC\n", + " )\n", + " AS row_number\n", + " FROM\n", + " int__resultats_udi_communes\n", + " WHERE\n", + " categorie = 'nitrate'\n", + " AND cdparametresiseeaux = 'NO3'\n", + " AND -- On garde les prélèvements de moins d'un an\n", + " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + " AND cdreseau IN( '034005906')" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "30487685-f619-43ab-be20-054cb46db3c6", "metadata": {}, "outputs": [ @@ -715,7 +1011,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2f02427c65a2483982d14158533ba333", + "model_id": "44938b0061f2429fac4a8d6cbc417b43", "version_major": 2, "version_minor": 0 }, @@ -739,34 +1035,28 @@ " \n", " \n", " \n", - " donnee_manquante\n", - " 001000251\n", - " 976003881\n", + " sup_limite_qualite\n", + " 002000060\n", + " 089003505\n", " \n", " \n", - " conforme\n", + " inf_limite_qualite\n", " 001000003\n", - " 976003489\n", - " \n", - " \n", - " non_conforme\n", - " 002000060\n", - " 095000346\n", + " 976003881\n", " \n", " \n", "" ], "text/plain": [ - "+------------------+--------------+--------------+\n", - "| resultat | min_cdreseau | max_cdreseau |\n", - "+------------------+--------------+--------------+\n", - "| donnee_manquante | 001000251 | 976003881 |\n", - "| conforme | 001000003 | 976003489 |\n", - "| non_conforme | 002000060 | 095000346 |\n", - "+------------------+--------------+--------------+" + "+--------------------+--------------+--------------+\n", + "| resultat | min_cdreseau | max_cdreseau |\n", + "+--------------------+--------------+--------------+\n", + "| sup_limite_qualite | 002000060 | 089003505 |\n", + "| inf_limite_qualite | 001000003 | 976003881 |\n", + "+--------------------+--------------+--------------+" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -790,7 +1080,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "id": "761be547-f21e-4712-a898-4175f8faa89c", "metadata": {}, "outputs": [ @@ -809,7 +1099,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2f660f72adc44235bf8ad78007bab56c", + "model_id": "bb7505505f924276a448034699a521e1", "version_major": 2, "version_minor": 0 }, @@ -828,145 +1118,127 @@ " \n", " cdreseau\n", " categorie\n", - " periode\n", - " dernier_prel_datetime\n", " nb_parametres\n", - " valtraduite_no2\n", - " valtraduite_no3\n", + " dernier_prel_datetime\n", + " limite_qualite_no3\n", + " limite_qualite_no2\n", + " limite_qualite_no3_no2\n", " valtraduite_no3_no2\n", - " valtraduite_no3_no2_calc\n", - " date_diff_count\n", - " \n", - " \n", + " valtraduite_no3\n", + " valtraduite_no2\n", + " resultat\n", + " \n", + " \n", " \n", " \n", " 001000003\n", " nitrate\n", - " dernier_prel\n", - " 2025-01-21 12:35:00\n", " 3\n", - " 0.000\n", - " 14.000\n", + " 2025-01-21 12:35:00\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", " 0.280\n", - " 0.28\n", - " 0\n", + " 14.000\n", + " 0.000\n", + " inf_limite_qualite\n", " \n", " \n", " 001000260\n", " nitrate\n", - " dernier_prel\n", - " 2024-12-06 12:02:00\n", " 3\n", - " 0.000\n", - " 2.400\n", + " 2024-12-06 12:02:00\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", " 0.050\n", - " 0.048\n", - " 0\n", + " 2.400\n", + " 0.000\n", + " inf_limite_qualite\n", " \n", " \n", " 001000268\n", " nitrate\n", - " dernier_prel\n", - " 2025-01-13 09:57:00\n", " 3\n", - " 0.000\n", - " 6.000\n", + " 2025-01-13 09:57:00\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", " 0.120\n", - " 0.12\n", - " 0\n", + " 6.000\n", + " 0.000\n", + " inf_limite_qualite\n", " \n", " \n", " 001000293\n", " nitrate\n", - " dernier_prel\n", - " 2025-01-28 11:29:00\n", " 3\n", - " 0.000\n", - " 3.700\n", + " 2025-01-28 11:29:00\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", " 0.070\n", - " 0.07400000000000001\n", - " 0\n", + " 3.700\n", + " 0.000\n", + " inf_limite_qualite\n", " \n", " \n", " 002000060\n", " nitrate\n", - " dernier_prel\n", + " 2\n", " 2024-10-18 08:50:00\n", - " 3\n", - " 0.000\n", - " 59.300\n", - " 0.000\n", - " 1.186\n", - " 0\n", - " \n", - " \n", - " 073002059\n", - " nitrate\n", - " dernier_prel\n", - " 2024-10-23 09:28:00\n", - " 1\n", - " 0.000\n", - " None\n", - " None\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", " None\n", - " 0\n", - " \n", - " \n", - " 088002296\n", - " nitrate\n", - " dernier_prel\n", - " 2024-09-27 12:00:00\n", - " 1\n", + " 59.300\n", " 0.000\n", - " None\n", - " None\n", - " None\n", - " 0\n", + " sup_limite_qualite\n", " \n", " \n", " 095000346\n", " nitrate\n", - " dernier_prel\n", + " 1\n", " 2025-02-27 08:35:00\n", - " 3\n", - " 0.000\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", + " None\n", " 47.000\n", - " 1.000\n", - " 0.94\n", - " 241\n", + " None\n", + " inf_limite_qualite\n", " \n", " \n", " 973000028\n", " nitrate\n", - " dernier_prel\n", - " 2024-12-02 11:30:00\n", " 2\n", + " 2024-12-02 11:30:00\n", + " 50.0\n", + " 0.5\n", + " 1.0\n", + " None\n", " 0.000\n", " 0.000\n", - " None\n", - " 0.0\n", - " 0\n", + " inf_limite_qualite\n", " \n", " \n", "" ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+-----------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | valtraduite_no2 | valtraduite_no3 | valtraduite_no3_no2 | valtraduite_no3_no2_calc | date_diff_count |\n", - "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+-----------------+\n", - "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | 0.000 | 14.000 | 0.280 | 0.28 | 0 |\n", - "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | 0.000 | 2.400 | 0.050 | 0.048 | 0 |\n", - "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | 0.000 | 6.000 | 0.120 | 0.12 | 0 |\n", - "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | 0.000 | 3.700 | 0.070 | 0.07400000000000001 | 0 |\n", - "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | 0.000 | 59.300 | 0.000 | 1.186 | 0 |\n", - "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | 0.000 | None | None | None | 0 |\n", - "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | 0.000 | None | None | None | 0 |\n", - "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | 0.000 | 47.000 | 1.000 | 0.94 | 241 |\n", - "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | 0.000 | 0.000 | None | 0.0 | 0 |\n", - "+-----------+-----------+--------------+-----------------------+---------------+-----------------+-----------------+---------------------+--------------------------+-----------------+" + "+-----------+-----------+---------------+-----------------------+--------------------+--------------------+------------------------+---------------------+-----------------+-----------------+--------------------+\n", + "| cdreseau | categorie | nb_parametres | dernier_prel_datetime | limite_qualite_no3 | limite_qualite_no2 | limite_qualite_no3_no2 | valtraduite_no3_no2 | valtraduite_no3 | valtraduite_no2 | resultat |\n", + "+-----------+-----------+---------------+-----------------------+--------------------+--------------------+------------------------+---------------------+-----------------+-----------------+--------------------+\n", + "| 001000003 | nitrate | 3 | 2025-01-21 12:35:00 | 50.0 | 0.5 | 1.0 | 0.280 | 14.000 | 0.000 | inf_limite_qualite |\n", + "| 001000260 | nitrate | 3 | 2024-12-06 12:02:00 | 50.0 | 0.5 | 1.0 | 0.050 | 2.400 | 0.000 | inf_limite_qualite |\n", + "| 001000268 | nitrate | 3 | 2025-01-13 09:57:00 | 50.0 | 0.5 | 1.0 | 0.120 | 6.000 | 0.000 | inf_limite_qualite |\n", + "| 001000293 | nitrate | 3 | 2025-01-28 11:29:00 | 50.0 | 0.5 | 1.0 | 0.070 | 3.700 | 0.000 | inf_limite_qualite |\n", + "| 002000060 | nitrate | 2 | 2024-10-18 08:50:00 | 50.0 | 0.5 | 1.0 | None | 59.300 | 0.000 | sup_limite_qualite |\n", + "| 095000346 | nitrate | 1 | 2025-02-27 08:35:00 | 50.0 | 0.5 | 1.0 | None | 47.000 | None | inf_limite_qualite |\n", + "| 973000028 | nitrate | 2 | 2024-12-02 11:30:00 | 50.0 | 0.5 | 1.0 | None | 0.000 | 0.000 | inf_limite_qualite |\n", + "+-----------+-----------+---------------+-----------------------+--------------------+--------------------+------------------------+---------------------+-----------------+-----------------+--------------------+" ] }, - "execution_count": 8, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -977,11 +1249,8 @@ "last_pvl AS (\n", " SELECT\n", " cdreseau,\n", - " categorie,\n", - " cdparametresiseeaux,\n", - " limite_qualite,\n", + " referenceprel,\n", " datetimeprel,\n", - " valtraduite,\n", " ROW_NUMBER()\n", " OVER (\n", " PARTITION BY cdreseau, cdparametresiseeaux\n", @@ -989,17 +1258,18 @@ " )\n", " AS row_number\n", " FROM\n", - " int__resultats_udi_communes\n", + " int__resultats_udi_communes\n", " WHERE\n", " categorie = 'nitrate'\n", - " AND\n", - " -- On garde les prélèvements de moins d'un an\n", + " AND cdparametresiseeaux = 'NO3'\n", + " AND -- On garde les prélèvements de moins d'un an\n", " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", + " AND valtraduite IS NOT NULL \n", "),\n", "\n", "valeur_ref AS (\n", - "SELECT \n", - " MAX(\n", + " SELECT\n", + " MAX(\n", " CASE\n", " WHEN\n", " cdparametresiseeaux = 'NO3'\n", @@ -1020,115 +1290,139 @@ " THEN limite_qualite\n", " END\n", " ) AS limite_qualite_no2\n", - "FROM int__valeurs_de_reference\n", - "WHERE categorie_1='nitrate' \n", + " FROM\n", + " int__valeurs_de_reference\n", + " WHERE\n", + " categorie_1 = 'nitrate'\n", "),\n", "\n", "split_nitrites AS (\n", " SELECT\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie,\n", - " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", - " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_n03,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_n02,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_no3_no2,\n", + " resultats.cdreseau,\n", + " resultats.categorie,\n", + " resultats.datetimeprel AS dernier_prel_datetime,\n", + " COUNT(DISTINCT resultats.cdparametresiseeaux) AS nb_parametres,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.valtraduite\n", + " resultats.cdparametresiseeaux = 'NO3'\n", + " THEN resultats.valtraduite\n", " END\n", " ) AS valtraduite_no3,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.valtraduite\n", + " resultats.cdparametresiseeaux = 'NO2'\n", + " THEN resultats.valtraduite\n", " END\n", " ) AS valtraduite_no2,\n", " MAX(\n", " CASE\n", " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", - " THEN last_pvl.valtraduite\n", + " resultats.cdparametresiseeaux = 'NO3_NO2'\n", + " THEN resultats.valtraduite\n", " END\n", " ) AS valtraduite_no3_no2\n", " FROM\n", " last_pvl\n", + " LEFT JOIN\n", + " int__resultats_udi_communes AS resultats\n", + " ON\n", + " resultats.cdreseau = last_pvl.cdreseau\n", + " AND resultats.referenceprel = last_pvl.referenceprel\n", " WHERE\n", - " last_pvl.row_number = 1\n", + " last_pvl.row_number = 1 \n", + " -- on ne veut que le dernier prélèvement\n", + " AND resultats.categorie = 'nitrate' \n", + " -- on ne veut que les résultats associé aux nitrates/nitrites\n", " GROUP BY\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie\n", + " resultats.cdreseau,\n", + " resultats.categorie,\n", + " resultats.datetimeprel\n", "),\n", "\n", "split_nitrites_with_ref AS (\n", - " SELECT \n", - " split_nitrites.*,\n", + " SELECT\n", + " split_nitrites.cdreseau,\n", + " split_nitrites.categorie,\n", + " split_nitrites.nb_parametres,\n", + " split_nitrites.dernier_prel_datetime,\n", " valeur_ref.limite_qualite_no3,\n", - " valeur_ref.limite_qualite_no3_no2,\n", " valeur_ref.limite_qualite_no2,\n", - " split_nitrites.valtraduite_no3 / 50\n", - " + split_nitrites.valtraduite_no2 / 3 AS valtraduite_no3_no2_calc,\n", - " DATE_DIFF(\n", - " 'day',\n", - " LEAST(\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02,\n", - " split_nitrites.dernier_prel_datetime_no3_no2\n", - " ),\n", - " GREATEST(\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02,\n", - " split_nitrites.dernier_prel_datetime_no3_no2\n", - " )\n", - " ) AS date_diff_count\n", - " FROM split_nitrites\n", - " CROSS JOIN valeur_ref\n", - "\n", - ")\n", - "\n", - "SELECT\n", - " split_nitrites_with_ref.cdreseau,\n", - " split_nitrites_with_ref.categorie,\n", - " 'dernier_prel' AS periode,\n", - " split_nitrites_with_ref.dernier_prel_datetime,\n", - " split_nitrites_with_ref.nb_parametres,\n", - " split_nitrites_with_ref.valtraduite_no2,\n", - " split_nitrites_with_ref.valtraduite_no3,\n", - " split_nitrites_with_ref.valtraduite_no3_no2,\n", - " split_nitrites_with_ref.valtraduite_no3_no2_calc,\n", - " split_nitrites_with_ref.date_diff_count\n", + " valeur_ref.limite_qualite_no3_no2,\n", + " split_nitrites.valtraduite_no3_no2,\n", + " split_nitrites.valtraduite_no3,\n", + " split_nitrites.valtraduite_no2\n", + " FROM\n", + " split_nitrites\n", + " CROSS JOIN\n", + " valeur_ref\n", + ") \n", + " \n", + "SELECT \n", + " *,\n", + " CASE\n", + " WHEN -- Si nitrates (no3) et pas nitrites (no2)\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL \n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NULL\n", + " AND\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", + " THEN 'inf_limite_qualite'\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL \n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NULL\n", + " AND\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " >= split_nitrites_with_ref.limite_qualite_no3\n", + " THEN 'sup_limite_qualite'\n", + " WHEN -- Si nitrates (no3) ET nitrites (no2)\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL\n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL\n", + " AND (\n", + " split_nitrites_with_ref.valtraduite_no3\n", + " >= split_nitrites_with_ref.limite_qualite_no3\n", + " OR split_nitrites_with_ref.valtraduite_no2\n", + " >= split_nitrites_with_ref.limite_qualite_no2\n", + " OR COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " (\n", + " split_nitrites_with_ref.valtraduite_no3 / 50\n", + " + split_nitrites_with_ref.valtraduite_no2 / 3\n", + " )\n", + " )\n", + " >= split_nitrites_with_ref.limite_qualite_no3_no2\n", + " )\n", + " THEN 'sup_limite_qualite'\n", + " WHEN\n", + " split_nitrites_with_ref.valtraduite_no3 IS NOT NULL\n", + " AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL\n", + " AND split_nitrites_with_ref.valtraduite_no3\n", + " < split_nitrites_with_ref.limite_qualite_no3\n", + " AND split_nitrites_with_ref.valtraduite_no2\n", + " < split_nitrites_with_ref.limite_qualite_no2\n", + " AND COALESCE(\n", + " split_nitrites_with_ref.valtraduite_no3_no2,\n", + " (\n", + " split_nitrites_with_ref.valtraduite_no3 / 50\n", + " + split_nitrites_with_ref.valtraduite_no2 / 3\n", + " )\n", + " )\n", + " < split_nitrites_with_ref.limite_qualite_no3_no2\n", + " THEN 'inf_limite_qualite'\n", + " ELSE 'erreur'\n", + " END AS resultat\n", "FROM\n", - " split_nitrites_with_ref \n", + " split_nitrites_with_ref\n", "WHERE\n", - " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293','008000681','008000546')\n", "ORDER BY \n", " cdreseau" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "id": "2137dd22-ec22-4b37-93ee-30b75c06ad42", "metadata": {}, "outputs": [ @@ -1147,7 +1441,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f22ecc549857467ca4ad388576cc30b4", + "model_id": "2ee8d4f0d01b40d18f8980a9126ab1b5", "version_major": 2, "version_minor": 0 }, @@ -1179,7 +1473,7 @@ " dernier_prel\n", " 2025-01-21 12:35:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", " 001000260\n", @@ -1187,7 +1481,7 @@ " dernier_prel\n", " 2024-12-06 12:02:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", " 001000268\n", @@ -1195,7 +1489,7 @@ " dernier_prel\n", " 2025-01-13 09:57:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", " 001000293\n", @@ -1203,39 +1497,39 @@ " dernier_prel\n", " 2025-01-28 11:29:00\n", " 3\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", " 002000060\n", " nitrate\n", " dernier_prel\n", " 2024-10-18 08:50:00\n", - " 3\n", - " non_conforme\n", + " 2\n", + " sup_limite_qualite\n", " \n", " \n", - " 073002059\n", + " 008000546\n", " nitrate\n", " dernier_prel\n", - " 2024-10-23 09:28:00\n", + " 2025-02-07 10:38:00\n", " 1\n", - " donnee_manquante\n", + " sup_limite_qualite\n", " \n", " \n", - " 088002296\n", + " 008000681\n", " nitrate\n", " dernier_prel\n", - " 2024-09-27 12:00:00\n", + " 2025-02-12 11:38:00\n", " 1\n", - " donnee_manquante\n", + " sup_limite_qualite\n", " \n", " \n", " 095000346\n", " nitrate\n", " dernier_prel\n", " 2025-02-27 08:35:00\n", - " 3\n", - " non_conforme\n", + " 1\n", + " inf_limite_qualite\n", " \n", " \n", " 973000028\n", @@ -1243,28 +1537,28 @@ " dernier_prel\n", " 2024-12-02 11:30:00\n", " 2\n", - " conforme\n", + " inf_limite_qualite\n", " \n", " \n", "" ], "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | conforme |\n", - "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", - "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | conforme |\n", - "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | conforme |\n", - "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", - "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", - "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | donnee_manquante |\n", - "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | non_conforme |\n", - "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+" + "+-----------+-----------+--------------+-----------------------+---------------+--------------------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------------+\n", + "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | inf_limite_qualite |\n", + "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | inf_limite_qualite |\n", + "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | inf_limite_qualite |\n", + "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | inf_limite_qualite |\n", + "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 2 | sup_limite_qualite |\n", + "| 008000546 | nitrate | dernier_prel | 2025-02-07 10:38:00 | 1 | sup_limite_qualite |\n", + "| 008000681 | nitrate | dernier_prel | 2025-02-12 11:38:00 | 1 | sup_limite_qualite |\n", + "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 1 | inf_limite_qualite |\n", + "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | inf_limite_qualite |\n", + "+-----------+-----------+--------------+-----------------------+---------------+--------------------+" ] }, - "execution_count": 9, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -1276,37 +1570,15 @@ " FROM\n", " int__resultats_nitrites_udi_dernier\n", "WHERE\n", - " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293',)\n", + " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293','008000681','008000546')\n", "ORDER BY \n", " cdreseau" ] }, - { - "cell_type": "markdown", - "id": "2cee74c0-c1e2-4b4e-9ccc-aef10805f621", - "metadata": {}, - "source": [ - "# Ajout d'une condition sur les dates de prélèvement/mesure\n", - "\n", - "**Les règles choisies**\n", - " \n", - "_Si on a les 3 paramètres disponibles **avec moins de 30 jours de différences**_\n", - "- Si Nitrates < 50 mg/L et nitrites < 0,5 mg/L et nitrate/50 + nitrites/3 < 1 mg/L --> CONFORME\n", - "- Si Nitrates >= 50 mg/L et/ou nitrites >= 0,5 mg/L et/ou nitrate/50 + nitrites/3 >= 1 mg/L --> NON CONFORME\n", - "\n", - "\n", - "_Si on n'a pas nitrate/50 + nitrites/3 mais qu'on a nitrate et nitrites_
\n", - "==> on le calcule et on applique comme ci-dessus\n", - "\n", - "- Si on n'a pas nitrate (no3) --> donnee_manquante\n", - "- Si on n'a pas nitrites (no2) et que nitrate >= 50 mg/L --> NON CONFORME\n", - "- Si on n'a pas nitrites (no2) et que nitrate < 50 mg/L --> donnee_manquant \n" - ] - }, { "cell_type": "code", - "execution_count": 10, - "id": "27337155-ed39-4db0-931a-197c9c055538", + "execution_count": 15, + "id": "4033d129-eeab-4ec9-ac40-39a4101e3083", "metadata": {}, "outputs": [ { @@ -1324,7 +1596,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6810d3c6d92c4864964687c62b4287c9", + "model_id": "bdc2fedf16ea411382a0c221d42c4667", "version_major": 2, "version_minor": 0 }, @@ -1341,1257 +1613,93 @@ "\n", " \n", " \n", - " \n", - " \n", - " \n", " \n", " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", " \n", - "
date_diffdernier_prel_datetime_n03dernier_prel_datetime_n02cdreseaucategorienb_parametresperiodedernier_prel_datetimevaltraduite_no3valtraduite_no2valtraduite_no3_no2date_diff_countnb_parametresresultat
-452024-11-25 09:29:002024-10-11 12:55:00002000074nitrate32024-11-25 09:29:0056.900None0.045
-612024-12-23 13:09:002024-10-23 08:29:00002001241nitrate32024-12-23 13:09:0052.500None0.061
-1412025-01-28 09:24:002024-09-09 09:09:00008000002nitrate22025-01-28 09:24:0052.000None1.04141
-1612025-02-07 10:38:002024-08-30 13:21:00008000546nitrate22025-02-07 10:38:0076.300None1.526161
-372025-02-26 11:59:002025-01-20 13:30:00008000679nitrate32025-02-26 11:59:0053.700None0.037
-1002025-02-12 11:38:002024-11-04 11:23:00008000681nitrate22025-02-12 11:38:0054.900None1.0979999999999999100
-1422025-02-19 12:10:002024-09-30 08:32:00008001171nitrate22025-02-19 12:10:0050.500None1.01142
-742025-02-18 10:16:002024-12-06 08:10:00010000072nitrate32025-02-18 10:16:0056.000None1.174
-1752025-02-18 12:24:002024-08-27 10:03:00010000122nitrate32025-02-18 12:24:0064.000None1.1175
-982025-02-24 11:25:002024-11-18 11:22:00010000125nitrate32025-02-24 11:25:0072.000None1.498
\n", - "Truncated to displaylimit of 10." + "" ], "text/plain": [ - "+-----------+---------------------------+---------------------------+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+-----------------+\n", - "| date_diff | dernier_prel_datetime_n03 | dernier_prel_datetime_n02 | cdreseau | categorie | nb_parametres | dernier_prel_datetime | valtraduite_no3 | valtraduite_no2 | valtraduite_no3_no2 | date_diff_count |\n", - "+-----------+---------------------------+---------------------------+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+-----------------+\n", - "| -45 | 2024-11-25 09:29:00 | 2024-10-11 12:55:00 | 002000074 | nitrate | 3 | 2024-11-25 09:29:00 | 56.900 | None | 0.0 | 45 |\n", - "| -61 | 2024-12-23 13:09:00 | 2024-10-23 08:29:00 | 002001241 | nitrate | 3 | 2024-12-23 13:09:00 | 52.500 | None | 0.0 | 61 |\n", - "| -141 | 2025-01-28 09:24:00 | 2024-09-09 09:09:00 | 008000002 | nitrate | 2 | 2025-01-28 09:24:00 | 52.000 | None | 1.04 | 141 |\n", - "| -161 | 2025-02-07 10:38:00 | 2024-08-30 13:21:00 | 008000546 | nitrate | 2 | 2025-02-07 10:38:00 | 76.300 | None | 1.526 | 161 |\n", - "| -37 | 2025-02-26 11:59:00 | 2025-01-20 13:30:00 | 008000679 | nitrate | 3 | 2025-02-26 11:59:00 | 53.700 | None | 0.0 | 37 |\n", - "| -100 | 2025-02-12 11:38:00 | 2024-11-04 11:23:00 | 008000681 | nitrate | 2 | 2025-02-12 11:38:00 | 54.900 | None | 1.0979999999999999 | 100 |\n", - "| -142 | 2025-02-19 12:10:00 | 2024-09-30 08:32:00 | 008001171 | nitrate | 2 | 2025-02-19 12:10:00 | 50.500 | None | 1.01 | 142 |\n", - "| -74 | 2025-02-18 10:16:00 | 2024-12-06 08:10:00 | 010000072 | nitrate | 3 | 2025-02-18 10:16:00 | 56.000 | None | 1.1 | 74 |\n", - "| -175 | 2025-02-18 12:24:00 | 2024-08-27 10:03:00 | 010000122 | nitrate | 3 | 2025-02-18 12:24:00 | 64.000 | None | 1.1 | 175 |\n", - "| -98 | 2025-02-24 11:25:00 | 2024-11-18 11:22:00 | 010000125 | nitrate | 3 | 2025-02-24 11:25:00 | 72.000 | None | 1.4 | 98 |\n", - "+-----------+---------------------------+---------------------------+-----------+-----------+---------------+-----------------------+-----------------+-----------------+---------------------+-----------------+\n", - "Truncated to displaylimit of 10." + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", + "+----------+-----------+---------+-----------------------+---------------+----------+\n", + "+----------+-----------+---------+-----------------------+---------------+----------+" ] }, - "execution_count": 10, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%sql\n", - "WITH\n", - "last_pvl AS (\n", - " SELECT\n", - " cdreseau,\n", - " categorie,\n", - " cdparametresiseeaux,\n", - " limite_qualite,\n", - " datetimeprel,\n", - " valtraduite,\n", - " ROW_NUMBER()\n", - " OVER (\n", - " PARTITION BY cdreseau, cdparametresiseeaux\n", - " ORDER BY datetimeprel DESC\n", - " )\n", - " AS row_number\n", - " FROM\n", - " int__resultats_udi_communes\n", - " WHERE\n", - " categorie = 'nitrate'\n", - " AND\n", - " -- On garde les prélèvements de moins d'un an\n", - " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", - "),\n", - "\n", - "valeur_ref AS (\n", - "SELECT \n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " cdparametresiseeaux = 'NO3'\n", - " THEN limite_qualite\n", - " END\n", - " ) AS limite_qualite_no3,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " cdparametresiseeaux = 'NO3_NO2'\n", - " THEN limite_qualite\n", - " END\n", - " ) AS limite_qualite_no3_no2,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " cdparametresiseeaux = 'NO2'\n", - " THEN limite_qualite\n", - " END\n", - " ) AS limite_qualite_no2\n", - "FROM int__valeurs_de_reference\n", - "WHERE categorie_1='nitrate' \n", - "),\n", - "\n", - "split_nitrites AS (\n", - " SELECT\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie,\n", - " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", - " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_n03,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_n02,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.valtraduite\n", - " END\n", - " ) AS valtraduite_no3,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.valtraduite\n", - " END\n", - " ) AS valtraduite_no2,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", - " THEN last_pvl.valtraduite\n", - " END\n", - " ) AS valtraduite_no3_no2\n", - " FROM\n", - " last_pvl\n", - " WHERE\n", - " last_pvl.row_number = 1\n", - " GROUP BY\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie\n", - "),\n", - "\n", - "split_nitrites_with_ref AS (\n", - " SELECT \n", - " DATE_DIFF( 'day',split_nitrites.dernier_prel_datetime_n03,split_nitrites.dernier_prel_datetime_n02) AS date_diff,\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02,\n", - " split_nitrites.cdreseau,\n", - " split_nitrites.categorie,\n", - " split_nitrites.nb_parametres,\n", - " split_nitrites.dernier_prel_datetime,\n", - " IF ( DATE_DIFF( 'day',split_nitrites.dernier_prel_datetime_n03,split_nitrites.dernier_prel_datetime_n02) >30,\n", - " NULL, valtraduite_no3) AS valtraduite_no3,\n", - " IF ( DATE_DIFF( 'day',split_nitrites.dernier_prel_datetime_n02,split_nitrites.dernier_prel_datetime_n03) >30,\n", - " NULL, valtraduite_no2) AS valtraduite_no2,\n", - " COALESCE(\n", - " split_nitrites.valtraduite_no3_no2,\n", - " (\n", - " split_nitrites.valtraduite_no3 / 50\n", - " + split_nitrites.valtraduite_no2 / 3\n", - " )\n", - " ) AS valtraduite_no3_no2,\n", - " \n", - " DATE_DIFF(\n", - " 'day',\n", - " LEAST(\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02\n", - " ),\n", - " GREATEST(\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02\n", - " )\n", - " ) AS date_diff_count\n", - " FROM split_nitrites\n", - " CROSS JOIN valeur_ref\n", - "\n", - ")\n", - "\n", - "SELECT\n", - "*\n", - "FROM\n", - " split_nitrites_with_ref \n", - "WHERE\n", - " split_nitrites_with_ref.date_diff_count > 30\n", - " AND valtraduite_no3 > 50\n", - "ORDER BY \n", - " cdreseau" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "dbdcec00-8e5c-4eb0-9fac-2a5f90709383", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ], - "text/plain": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0bc8fea3bd484446ae765a59dfa149bb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000311nitratedernier_prel2024-12-10 12:02:003conforme
001000336nitratedernier_prel2024-10-31 10:45:003conforme
001000373nitratedernier_prel2024-11-29 11:03:003conforme
001000386nitratedernier_prel2024-11-27 08:19:003conforme
001000388nitratedernier_prel2024-06-05 08:47:003conforme
001000447nitratedernier_prel2025-01-16 09:17:003conforme
001000498nitratedernier_prel2024-06-03 08:15:003conforme
001000525nitratedernier_prel2025-02-03 10:21:003conforme
001000584nitratedernier_prel2025-01-09 10:01:003donnee_manquante
001000655nitratedernier_prel2025-01-27 09:48:003conforme
\n", - "Truncated to displaylimit of 10." - ], - "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000311 | nitrate | dernier_prel | 2024-12-10 12:02:00 | 3 | conforme |\n", - "| 001000336 | nitrate | dernier_prel | 2024-10-31 10:45:00 | 3 | conforme |\n", - "| 001000373 | nitrate | dernier_prel | 2024-11-29 11:03:00 | 3 | conforme |\n", - "| 001000386 | nitrate | dernier_prel | 2024-11-27 08:19:00 | 3 | conforme |\n", - "| 001000388 | nitrate | dernier_prel | 2024-06-05 08:47:00 | 3 | conforme |\n", - "| 001000447 | nitrate | dernier_prel | 2025-01-16 09:17:00 | 3 | conforme |\n", - "| 001000498 | nitrate | dernier_prel | 2024-06-03 08:15:00 | 3 | conforme |\n", - "| 001000525 | nitrate | dernier_prel | 2025-02-03 10:21:00 | 3 | conforme |\n", - "| 001000584 | nitrate | dernier_prel | 2025-01-09 10:01:00 | 3 | donnee_manquante |\n", - "| 001000655 | nitrate | dernier_prel | 2025-01-27 09:48:00 | 3 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "Truncated to displaylimit of 10." - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%sql --save int__resultats_nitrites_udi_dernier_dateCheck\n", - "WITH\n", - "last_pvl AS (\n", - " SELECT\n", - " cdreseau,\n", - " categorie,\n", - " cdparametresiseeaux,\n", - " limite_qualite,\n", - " datetimeprel,\n", - " valtraduite,\n", - " ROW_NUMBER()\n", - " OVER (\n", - " PARTITION BY cdreseau, cdparametresiseeaux\n", - " ORDER BY datetimeprel DESC\n", - " )\n", - " AS row_number\n", - " FROM\n", - " int__resultats_udi_communes\n", - " WHERE\n", - " categorie = 'nitrate'\n", - " AND\n", - " -- On garde les prélèvements de moins d'un an\n", - " CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR\n", - "),\n", - "\n", - "valeur_ref AS (\n", - " SELECT\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " cdparametresiseeaux = 'NO3'\n", - " THEN limite_qualite\n", - " END\n", - " ) AS limite_qualite_no3,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " cdparametresiseeaux = 'NO3_NO2'\n", - " THEN limite_qualite\n", - " END\n", - " ) AS limite_qualite_no3_no2,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " cdparametresiseeaux = 'NO2'\n", - " THEN limite_qualite\n", - " END\n", - " ) AS limite_qualite_no2\n", - " FROM\n", - " int__valeurs_de_reference\n", - " WHERE\n", - " categorie_1 = 'nitrate'\n", - "),\n", - "\n", - "split_nitrites AS (\n", - " SELECT\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie,\n", - " COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres,\n", - " MAX(last_pvl.datetimeprel) AS dernier_prel_datetime,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.valtraduite\n", - " END\n", - " ) AS valtraduite_no3,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.valtraduite\n", - " END\n", - " ) AS valtraduite_no2,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3_NO2'\n", - " THEN last_pvl.valtraduite\n", - " END\n", - " ) AS valtraduite_no3_no2,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO3'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_n03,\n", - " MAX(\n", - " CASE\n", - " WHEN\n", - " last_pvl.cdparametresiseeaux = 'NO2'\n", - " THEN last_pvl.datetimeprel\n", - " END\n", - " ) AS dernier_prel_datetime_n02\n", - " FROM\n", - " last_pvl\n", - " WHERE\n", - " last_pvl.row_number = 1\n", - " GROUP BY\n", - " last_pvl.cdreseau,\n", - " last_pvl.categorie\n", - "),\n", - "\n", - "split_nitrites_with_ref AS (\n", - " SELECT\n", - " split_nitrites.cdreseau,\n", - " split_nitrites.categorie,\n", - " split_nitrites.nb_parametres,\n", - " split_nitrites.dernier_prel_datetime,\n", - " valeur_ref.limite_qualite_no3,\n", - " valeur_ref.limite_qualite_no2,\n", - " valeur_ref.limite_qualite_no3_no2,\n", - " split_nitrites.valtraduite_no3_no2,\n", - " -- Si les mesures sont espacées de plus de 30 jours \n", - " -- on garde que la plus récente uniquement \n", - " IF(\n", - " DATE_DIFF(\n", - " 'day',\n", - " split_nitrites.dernier_prel_datetime_n03,\n", - " split_nitrites.dernier_prel_datetime_n02\n", - " ) > 30,\n", - " NULL,\n", - " split_nitrites.valtraduite_no3\n", - " ) AS valtraduite_no3,\n", - " IF(\n", - " DATE_DIFF(\n", - " 'day',\n", - " split_nitrites.dernier_prel_datetime_n02,\n", - " split_nitrites.dernier_prel_datetime_n03\n", - " ) > 30,\n", - " NULL,\n", - " split_nitrites.valtraduite_no2\n", - " ) AS valtraduite_no2\n", - " FROM\n", - " split_nitrites\n", - " CROSS JOIN\n", - " valeur_ref\n", - ")\n", - "\n", - "SELECT\n", - " split_nitrites_with_ref.cdreseau,\n", - " split_nitrites_with_ref.categorie,\n", - " 'dernier_prel' AS periode,\n", - " split_nitrites_with_ref.dernier_prel_datetime,\n", - " split_nitrites_with_ref.nb_parametres,\n", - " CASE\n", - " WHEN\n", - " split_nitrites_with_ref.valtraduite_no3\n", - " < split_nitrites_with_ref.limite_qualite_no3\n", - " AND split_nitrites_with_ref.valtraduite_no2\n", - " < split_nitrites_with_ref.limite_qualite_no2\n", - " -- Si on n'a pas valtraduite_no3_no2, on le calcule\n", - " -- si l'une des valeurs de la somme est NULL le total sera NULL \n", - " AND COALESCE(\n", - " split_nitrites_with_ref.valtraduite_no3_no2,\n", - " (\n", - " split_nitrites_with_ref.valtraduite_no3 / 50\n", - " + split_nitrites_with_ref.valtraduite_no2 / 3\n", - " )\n", - " )\n", - " < split_nitrites_with_ref.limite_qualite_no3_no2\n", - " THEN 'conforme'\n", - " WHEN\n", - " split_nitrites_with_ref.valtraduite_no3\n", - " >= split_nitrites_with_ref.limite_qualite_no3\n", - " OR split_nitrites_with_ref.valtraduite_no2\n", - " >= split_nitrites_with_ref.limite_qualite_no2\n", - " -- Si on n'a pas valtraduite_no3_no2, on le calcule\n", - " -- si l'une des valeurs de la somme est NULL le total sera NULL \n", - " OR COALESCE(\n", - " split_nitrites_with_ref.valtraduite_no3_no2,\n", - " (\n", - " split_nitrites_with_ref.valtraduite_no3 / 50\n", - " + split_nitrites_with_ref.valtraduite_no2 / 3\n", - " )\n", - " )\n", - " >= split_nitrites_with_ref.limite_qualite_no3_no2\n", - " THEN 'non_conforme'\n", - " WHEN\n", - " split_nitrites_with_ref.valtraduite_no3 IS NULL\n", - " OR (\n", - " split_nitrites_with_ref.valtraduite_no2 IS NULL\n", - " AND split_nitrites_with_ref.valtraduite_no3\n", - " < split_nitrites_with_ref.limite_qualite_no3\n", - " )\n", - " THEN 'donnee_manquante'\n", - " ELSE 'error'\n", - " END AS resultat\n", - "FROM\n", - " split_nitrites_with_ref\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "3f32e544-93cb-4728-8015-b3183ec50f44", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ], - "text/plain": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "03a735b239374693b51eb407bbfba319", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
resultatcount_star()
conforme17329
non_conforme269
donnee_manquante4526
" - ], - "text/plain": [ - "+------------------+--------------+\n", - "| resultat | count_star() |\n", - "+------------------+--------------+\n", - "| conforme | 17329 |\n", - "| non_conforme | 269 |\n", - "| donnee_manquante | 4526 |\n", - "+------------------+--------------+" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck\n", - "SELECT \n", - " resultat, COUNT(*)\n", - "FROM \n", - " int__resultats_nitrites_udi_dernier_dateCheck\n", - "GROUP BY 1" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "1c02a4fb-edb6-4822-8a24-cedb82705f45", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ], - "text/plain": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e870dbf0ff544b178cb7231b4538d23e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
" - ], - "text/plain": [ - "+----------+-----------+---------+-----------------------+---------------+----------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+----------+-----------+---------+-----------------------+---------------+----------+\n", - "+----------+-----------+---------+-----------------------+---------------+----------+" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck\n", - "SELECT \n", - " * \n", - "FROM \n", - " int__resultats_nitrites_udi_dernier_dateCheck\n", - "WHERE \n", - " resultat='error'" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "674c3c38-693f-4025-86e4-554627d568ad", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ], - "text/plain": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "96da167393374630b0e28e87738828b0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000003nitratedernier_prel2025-01-21 12:35:003conforme
001000260nitratedernier_prel2024-12-06 12:02:003conforme
001000268nitratedernier_prel2025-01-13 09:57:003conforme
001000293nitratedernier_prel2025-01-28 11:29:003conforme
002000060nitratedernier_prel2024-10-18 08:50:003non_conforme
073002059nitratedernier_prel2024-10-23 09:28:001donnee_manquante
088002296nitratedernier_prel2024-09-27 12:00:001donnee_manquante
095000346nitratedernier_prel2025-02-27 08:35:003non_conforme
973000028nitratedernier_prel2024-12-02 11:30:002conforme
" - ], - "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000003 | nitrate | dernier_prel | 2025-01-21 12:35:00 | 3 | conforme |\n", - "| 001000260 | nitrate | dernier_prel | 2024-12-06 12:02:00 | 3 | conforme |\n", - "| 001000268 | nitrate | dernier_prel | 2025-01-13 09:57:00 | 3 | conforme |\n", - "| 001000293 | nitrate | dernier_prel | 2025-01-28 11:29:00 | 3 | conforme |\n", - "| 002000060 | nitrate | dernier_prel | 2024-10-18 08:50:00 | 3 | non_conforme |\n", - "| 073002059 | nitrate | dernier_prel | 2024-10-23 09:28:00 | 1 | donnee_manquante |\n", - "| 088002296 | nitrate | dernier_prel | 2024-09-27 12:00:00 | 1 | donnee_manquante |\n", - "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | non_conforme |\n", - "| 973000028 | nitrate | dernier_prel | 2024-12-02 11:30:00 | 2 | conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", - " SELECT \n", - " *\n", - " FROM\n", - " int__resultats_nitrites_udi_dernier_dateCheck\n", - "WHERE\n", - " cdreseau IN( '002000060','095000346','001000003','976003554','088002296','073002059','973000028','001000260','001000268','001000293')\n", - "ORDER BY \n", - " cdreseau" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "984e0235-3790-4da7-bb47-50e23e7fa61b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ], - "text/plain": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c1bc7e8aea3b422b88d2e462c8dd8ea4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
001000258nitratedernier_prel2025-01-21 09:45:003donnee_manquante
001000387nitratedernier_prel2025-02-13 10:48:003donnee_manquante
008000546nitratedernier_prel2025-02-07 10:38:002non_conforme
008000681nitratedernier_prel2025-02-12 11:38:002non_conforme
095000346nitratedernier_prel2025-02-27 08:35:003non_conforme
" - ], - "text/plain": [ - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+\n", - "| 001000258 | nitrate | dernier_prel | 2025-01-21 09:45:00 | 3 | donnee_manquante |\n", - "| 001000387 | nitrate | dernier_prel | 2025-02-13 10:48:00 | 3 | donnee_manquante |\n", - "| 008000546 | nitrate | dernier_prel | 2025-02-07 10:38:00 | 2 | non_conforme |\n", - "| 008000681 | nitrate | dernier_prel | 2025-02-12 11:38:00 | 2 | non_conforme |\n", - "| 095000346 | nitrate | dernier_prel | 2025-02-27 08:35:00 | 3 | non_conforme |\n", - "+-----------+-----------+--------------+-----------------------+---------------+------------------+" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", - " SELECT \n", - " *\n", - " FROM\n", - " int__resultats_nitrites_udi_dernier_dateCheck\n", - "WHERE\n", - " cdreseau IN('001000258','001000387','008000681','008000546','095000346')\n", - "ORDER BY \n", - " cdreseau" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "4033d129-eeab-4ec9-ac40-39a4101e3083", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ], - "text/plain": [ - "Running query in 'duckdb:///../../database/data.duckdb'" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ad9a179083cb44d38acae7d24f0b887f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cdreseaucategorieperiodedernier_prel_datetimenb_parametresresultat
" - ], - "text/plain": [ - "+----------+-----------+---------+-----------------------+---------------+----------+\n", - "| cdreseau | categorie | periode | dernier_prel_datetime | nb_parametres | resultat |\n", - "+----------+-----------+---------+-----------------------+---------------+----------+\n", - "+----------+-----------+---------+-----------------------+---------------+----------+" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "%%sql --with int__resultats_nitrites_udi_dernier_dateCheck \n", + "%%sql --with int__resultats_nitrites_udi_dernier\n", "-- dernier relevé\n", "SELECT\n", "*\n", "FROM\n", - " int__resultats_nitrites_udi_dernier_dateCheck\n", + " int__resultats_nitrites_udi_dernier\n", "WHERE\n", " (\n", " cdreseau = '001000003'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-01-21 12:35:00'\n", - " AND resultat != 'conforme'\n", + " AND resultat != 'inf_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '001000260'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2024-12-06 12:02:00'\n", - " AND resultat != 'conforme'\n", + " AND resultat != 'inf_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '001000268'\n", " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-12-06 12:02:00'\n", - " AND resultat != 'conforme'\n", + " AND dernier_prel_datetime = '2025-01-13 09:57:00'\n", + " AND resultat != 'inf_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '002000060'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2024-10-18 08:50:00'\n", - " AND resultat != 'non_conforme'\n", - " )\n", - " OR\n", - " (\n", - " cdreseau = '073002059'\n", - " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-10-23 09:28:00'\n", - " AND resultat != 'donnee_manquante'\n", - " )\n", - " OR\n", - " (\n", - " cdreseau = '088002296'\n", - " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2024-09-27 12:00:00'\n", - " AND resultat != 'donnee_manquante'\n", + " AND resultat != 'sup_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '095000346'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-02-27 08:35:00'\n", - " AND resultat != 'non_conforme'\n", + " AND resultat != 'inf_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '973000028'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2024-12-02 11:30:00'\n", - " AND resultat != 'conforme'\n", - " )\n", - " OR\n", - " (\n", - " cdreseau = '001000258'\n", - " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2025-01-21 09:45:00'\n", - " AND resultat != 'donnee_manquante'\n", - " )\n", - " OR\n", - " (\n", - " cdreseau = '001000387'\n", - " AND categorie = 'nitrate'\n", - " AND dernier_prel_datetime = '2025-02-13 10:48:00'\n", - " AND resultat != 'donnee_manquante'\n", + " AND resultat != 'inf_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '008000681'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-02-12 11:38:00'\n", - " AND resultat != 'non_conforme'\n", + " AND resultat != 'sup_limite_qualite'\n", " )\n", " OR\n", " (\n", " cdreseau = '008000546'\n", " AND categorie = 'nitrate'\n", " AND dernier_prel_datetime = '2025-02-07 10:38:00'\n", - " AND resultat != 'non_conforme'\n", - " )\n" + " AND resultat != 'sup_limite_qualite'\n", + " )" ] }, { diff --git a/dbt_/models/intermediate/_int__models.yml b/dbt_/models/intermediate/_int__models.yml index 5c45ad59..885acc84 100644 --- a/dbt_/models/intermediate/_int__models.yml +++ b/dbt_/models/intermediate/_int__models.yml @@ -136,6 +136,7 @@ models: - name: cdreseau description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null @@ -166,6 +167,7 @@ models: columns: - name: cdreseau description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null @@ -266,6 +268,7 @@ models: - name: cdreseau description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null @@ -375,6 +378,7 @@ models: columns: - name: cdreseau description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null - name: periode diff --git a/dbt_/models/intermediate/cvm/_int__cvm_models.yaml b/dbt_/models/intermediate/cvm/_int__cvm_models.yaml index a305fd85..7917f508 100644 --- a/dbt_/models/intermediate/cvm/_int__cvm_models.yaml +++ b/dbt_/models/intermediate/cvm/_int__cvm_models.yaml @@ -17,6 +17,7 @@ models: type: SMALLINT - name: categorie description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + type: VARCHAR tests: - accepted_values: values: ["cvm"] @@ -27,8 +28,10 @@ models: - dbt_utils.expression_is_true: expression: "LIKE 'bilan_annuel%'" - name: nb_depassements + description: "Nombre de prélèvements qui dépassent la valeur de référence" type: INT - name: nb_prelevements + description: "Nombre de prélèvements total" type: INT - name: ratio_depassements description: "Pourcentage de prélèvements qui dépassent la limite" @@ -47,26 +50,34 @@ models: - not_null - unique - name: categorie + description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + type: VARCHAR tests: - not_null - name: periode + description: "Période du résultat" type: VARCHAR tests: - accepted_values: values: ["dernier_prel"] - name: dernier_prel_datetime + description: "Date du dernier prélèvement considéré" type: TIMESTAMP tests: - not_null - dbt_utils.expression_is_true: expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" - name: dernier_prel_valeur + description: "valeur mesurée sur le dernier prélèvement considéré" type: FLOAT - name: nb_parametres + description: "nombre de paramètres mesurés sur le prélèvement pour la catégorie considérée" type: INT tests: - not_null - name: resultat + description: "classification de la substance pour le prélèvement" + type: VARCHAR tests: - not_null - accepted_values: @@ -85,6 +96,7 @@ models: columns: - name: cdreseau description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null - name: annee @@ -92,6 +104,7 @@ models: type: SMALLINT - name: categorie description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + type: VARCHAR tests: - accepted_values: values: ["cvm"] @@ -102,7 +115,11 @@ models: - dbt_utils.expression_is_true: expression: "LIKE 'bilan_annuel%'" - name: nb_depassements + description: "Nombre de prélèvements qui dépassent la valeur de référence" + type: INT - name: nb_prelevements + description: "Nombre de prélèvements total" + type: INT - name: ratio_depassements description: "Pourcentage de prélèvements qui dépassent la limite" type: FLOAT @@ -116,30 +133,40 @@ models: description: Resultats CVM par UDI pour la temporalité 'derniers prélèvements' columns: - name: cdreseau + description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null - unique - name: categorie + description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + type: VARCHAR tests: - not_null - name: periode + description: "Période du résultat" type: VARCHAR tests: - accepted_values: values: ["dernier_prel"] - name: dernier_prel_datetime + description: "Date du dernier prélèvement considéré" type: TIMESTAMP tests: - not_null - dbt_utils.expression_is_true: expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" - name: dernier_prel_valeur + description: "valeur mesurée sur le dernier prélèvement considéré" type: FLOAT - name: nb_parametres + description: "nombre de paramètres mesurés sur le prélèvement pour la catégorie considérée" type: INT tests: - not_null - name: resultat + description: "classification de la substance pour le prélèvement" + type: VARCHAR tests: - not_null - accepted_values: diff --git a/dbt_/models/intermediate/cvm/int__resultats_cvm_commune_dernier.sql b/dbt_/models/intermediate/cvm/int__resultats_cvm_commune_dernier.sql index 9a4c39be..c43640d1 100644 --- a/dbt_/models/intermediate/cvm/int__resultats_cvm_commune_dernier.sql +++ b/dbt_/models/intermediate/cvm/int__resultats_cvm_commune_dernier.sql @@ -40,7 +40,7 @@ SELECT WHEN last_pvl.valtraduite < last_pvl.limite_qualite THEN 'inf_0_5' - ELSE 'error' + ELSE 'erreur' END AS resultat FROM last_pvl diff --git a/dbt_/models/intermediate/cvm/int__resultats_cvm_udi_dernier.sql b/dbt_/models/intermediate/cvm/int__resultats_cvm_udi_dernier.sql index f0a27007..c553b684 100644 --- a/dbt_/models/intermediate/cvm/int__resultats_cvm_udi_dernier.sql +++ b/dbt_/models/intermediate/cvm/int__resultats_cvm_udi_dernier.sql @@ -40,7 +40,7 @@ SELECT WHEN last_pvl.valtraduite < last_pvl.limite_qualite THEN 'inf_0_5' - ELSE 'error' + ELSE 'erreur' END AS resultat FROM last_pvl diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index bfe74325..1e35efe6 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -5,36 +5,44 @@ models: description: Resultats nitrites par UDI pour la temporalité 'derniers prélèvements' columns: - name: cdreseau + description: "Code de l'installation (unité de distribution)" + type: VARCHAR tests: - not_null - unique - name: categorie + description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + type: VARCHAR tests: - not_null - accepted_values: values: ["nitrate"] - name: periode + description: "Période du résultat" type: VARCHAR tests: - accepted_values: values: ["dernier_prel"] - name: dernier_prel_datetime + description: "Date du dernier prélèvement considéré" type: TIMESTAMP tests: - not_null - dbt_utils.expression_is_true: expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" - name: dernier_prel_valeur + description: "valeur mesurée sur le dernier prélèvement considéré" type: FLOAT - name: nb_parametres + description: "nombre de paramètres mesurés sur le prélèvement pour la catégorie considérée" type: INT tests: - not_null - name: resultat + description: "classification de la substance pour le prélèvement" tests: - not_null - accepted_values: values: - - donnee_manquante - - conforme - - non_conforme \ No newline at end of file + - inf_limite_qualite + - sup_limite_qualite \ No newline at end of file diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql index a632daf1..baf00075 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql @@ -2,11 +2,8 @@ WITH last_pvl AS ( SELECT cdreseau, - categorie, - cdparametresiseeaux, - limite_qualite, + referenceprel, datetimeprel, - valtraduite, ROW_NUMBER() OVER ( PARTITION BY cdreseau, cdparametresiseeaux @@ -17,9 +14,11 @@ last_pvl AS ( {{ ref('int__resultats_udi_communes') }} WHERE categorie = 'nitrate' - AND - -- On garde les prélèvements de moins d'un an + AND cdparametresiseeaux = 'NO3' + AND -- On garde les prélèvements de moins d'un an CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR + AND valtraduite IS NOT NULL + -- Cf cas cdreseau IN( '034005906') , referenceprel= 03400327764 ), valeur_ref AS ( @@ -53,52 +52,47 @@ valeur_ref AS ( split_nitrites AS ( SELECT - last_pvl.cdreseau, - last_pvl.categorie, - COUNT(DISTINCT last_pvl.cdparametresiseeaux) AS nb_parametres, - MAX(last_pvl.datetimeprel) AS dernier_prel_datetime, + resultats.cdreseau, + resultats.categorie, + resultats.datetimeprel AS dernier_prel_datetime, + COUNT(DISTINCT resultats.cdparametresiseeaux) AS nb_parametres, MAX( CASE WHEN - last_pvl.cdparametresiseeaux = 'NO3' - THEN last_pvl.valtraduite + resultats.cdparametresiseeaux = 'NO3' + THEN resultats.valtraduite END ) AS valtraduite_no3, MAX( CASE WHEN - last_pvl.cdparametresiseeaux = 'NO2' - THEN last_pvl.valtraduite + resultats.cdparametresiseeaux = 'NO2' + THEN resultats.valtraduite END ) AS valtraduite_no2, MAX( CASE WHEN - last_pvl.cdparametresiseeaux = 'NO3_NO2' - THEN last_pvl.valtraduite + resultats.cdparametresiseeaux = 'NO3_NO2' + THEN resultats.valtraduite END - ) AS valtraduite_no3_no2, - MAX( - CASE - WHEN - last_pvl.cdparametresiseeaux = 'NO3' - THEN last_pvl.datetimeprel - END - ) AS dernier_prel_datetime_n03, - MAX( - CASE - WHEN - last_pvl.cdparametresiseeaux = 'NO2' - THEN last_pvl.datetimeprel - END - ) AS dernier_prel_datetime_n02 + ) AS valtraduite_no3_no2 FROM last_pvl + LEFT JOIN + {{ ref('int__resultats_udi_communes') }} AS resultats + ON + last_pvl.cdreseau = resultats.cdreseau + AND last_pvl.referenceprel = resultats.referenceprel WHERE last_pvl.row_number = 1 + -- on ne veut que le dernier prélèvement + AND resultats.categorie = 'nitrate' + -- on ne veut que les résultats associé aux nitrates/nitrites GROUP BY - last_pvl.cdreseau, - last_pvl.categorie + resultats.cdreseau, + resultats.categorie, + resultats.datetimeprel ), split_nitrites_with_ref AS ( @@ -111,26 +105,8 @@ split_nitrites_with_ref AS ( valeur_ref.limite_qualite_no2, valeur_ref.limite_qualite_no3_no2, split_nitrites.valtraduite_no3_no2, - -- Si les mesures sont espacées de plus de 30 jours - -- on garde que la plus récente uniquement - IF( - DATE_DIFF( - 'day', - split_nitrites.dernier_prel_datetime_n03, - split_nitrites.dernier_prel_datetime_n02 - ) > 30, - NULL, - split_nitrites.valtraduite_no3 - ) AS valtraduite_no3, - IF( - DATE_DIFF( - 'day', - split_nitrites.dernier_prel_datetime_n02, - split_nitrites.dernier_prel_datetime_n03 - ) > 30, - NULL, - split_nitrites.valtraduite_no2 - ) AS valtraduite_no2 + split_nitrites.valtraduite_no3, + split_nitrites.valtraduite_no2 FROM split_nitrites CROSS JOIN @@ -144,13 +120,45 @@ SELECT split_nitrites_with_ref.dernier_prel_datetime, split_nitrites_with_ref.nb_parametres, CASE + WHEN -- Si nitrates (no3) et pas nitrites (no2) + split_nitrites_with_ref.valtraduite_no3 IS NOT NULL + AND split_nitrites_with_ref.valtraduite_no2 IS NULL + AND + split_nitrites_with_ref.valtraduite_no3 + < split_nitrites_with_ref.limite_qualite_no3 + THEN 'inf_limite_qualite' WHEN + split_nitrites_with_ref.valtraduite_no3 IS NOT NULL + AND split_nitrites_with_ref.valtraduite_no2 IS NULL + AND split_nitrites_with_ref.valtraduite_no3 + >= split_nitrites_with_ref.limite_qualite_no3 + THEN 'sup_limite_qualite' + WHEN -- Si nitrates (no3) ET nitrites (no2) + split_nitrites_with_ref.valtraduite_no3 IS NOT NULL + AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL + AND ( + split_nitrites_with_ref.valtraduite_no3 + >= split_nitrites_with_ref.limite_qualite_no3 + OR split_nitrites_with_ref.valtraduite_no2 + >= split_nitrites_with_ref.limite_qualite_no2 + OR COALESCE( + split_nitrites_with_ref.valtraduite_no3_no2, + ( + split_nitrites_with_ref.valtraduite_no3 / 50 + + split_nitrites_with_ref.valtraduite_no2 / 3 + ) + ) + >= split_nitrites_with_ref.limite_qualite_no3_no2 + ) + THEN 'sup_limite_qualite' + WHEN + split_nitrites_with_ref.valtraduite_no3 IS NOT NULL + AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL + AND split_nitrites_with_ref.valtraduite_no3 < split_nitrites_with_ref.limite_qualite_no3 AND split_nitrites_with_ref.valtraduite_no2 < split_nitrites_with_ref.limite_qualite_no2 - -- Si on n'a pas valtraduite_no3_no2, on le calcule - -- si l'une des valeurs de la somme est NULL le total sera NULL AND COALESCE( split_nitrites_with_ref.valtraduite_no3_no2, ( @@ -159,32 +167,8 @@ SELECT ) ) < split_nitrites_with_ref.limite_qualite_no3_no2 - THEN 'conforme' - WHEN - split_nitrites_with_ref.valtraduite_no3 - >= split_nitrites_with_ref.limite_qualite_no3 - OR split_nitrites_with_ref.valtraduite_no2 - >= split_nitrites_with_ref.limite_qualite_no2 - -- Si on n'a pas valtraduite_no3_no2, on le calcule - -- si l'une des valeurs de la somme est NULL le total sera NULL - OR COALESCE( - split_nitrites_with_ref.valtraduite_no3_no2, - ( - split_nitrites_with_ref.valtraduite_no3 / 50 - + split_nitrites_with_ref.valtraduite_no2 / 3 - ) - ) - >= split_nitrites_with_ref.limite_qualite_no3_no2 - THEN 'non_conforme' - WHEN - split_nitrites_with_ref.valtraduite_no3 IS NULL - OR ( - split_nitrites_with_ref.valtraduite_no2 IS NULL - AND split_nitrites_with_ref.valtraduite_no3 - < split_nitrites_with_ref.limite_qualite_no3 - ) - THEN 'donnee_manquante' - ELSE 'error' + THEN 'inf_limite_qualite' + ELSE 'erreur' END AS resultat FROM split_nitrites_with_ref diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index a72ce88a..1598fb41 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -14,82 +14,54 @@ WHERE cdreseau = '001000003' AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-01-21 12:35:00' - AND resultat != 'conforme' + AND resultat != 'inf_limite_qualite' ) OR ( cdreseau = '001000260' AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-12-06 12:02:00' - AND resultat != 'conforme' + AND resultat != 'inf_limite_qualite' ) OR ( cdreseau = '001000268' AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-12-06 12:02:00' - AND resultat != 'conforme' + AND dernier_prel_datetime = '2025-01-13 09:57:00' + AND resultat != 'inf_limite_qualite' ) OR ( cdreseau = '002000060' AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-10-18 08:50:00' - AND resultat != 'non_conforme' - ) - OR - ( - cdreseau = '073002059' - AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-10-23 09:28:00' - AND resultat != 'donnee_manquante' - ) - OR - ( - cdreseau = '088002296' - AND categorie = 'nitrate' - AND dernier_prel_datetime = '2024-09-27 12:00:00' - AND resultat != 'donnee_manquante' + AND resultat != 'sup_limite_qualite' ) OR ( cdreseau = '095000346' AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-02-27 08:35:00' - AND resultat != 'non_conforme' + AND resultat != 'inf_limite_qualite' ) OR ( cdreseau = '973000028' AND categorie = 'nitrate' AND dernier_prel_datetime = '2024-12-02 11:30:00' - AND resultat != 'conforme' - ) - OR - ( - cdreseau = '001000258' - AND categorie = 'nitrate' - AND dernier_prel_datetime = '2025-01-21 09:45:00' - AND resultat != 'donnee_manquante' - ) - OR - ( - cdreseau = '001000387' - AND categorie = 'nitrate' - AND dernier_prel_datetime = '2025-02-13 10:48:00' - AND resultat != 'donnee_manquante' + AND resultat != 'inf_limite_qualite' ) OR ( cdreseau = '008000681' AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-02-12 11:38:00' - AND resultat != 'non_conforme' + AND resultat != 'sup_limite_qualite' ) OR ( cdreseau = '008000546' AND categorie = 'nitrate' AND dernier_prel_datetime = '2025-02-07 10:38:00' - AND resultat != 'non_conforme' + AND resultat != 'sup_limite_qualite' ) From bf563075ab8bcd990ecf1a2b35ed3deda3c7c8e9 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Thu, 17 Apr 2025 11:54:01 +0200 Subject: [PATCH 19/30] fix --- dbt_/models/intermediate/int__union_resultats_udi.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index 6f7fac1f..42b496b2 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -85,8 +85,9 @@ SELECT null AS nb_prelevements, null AS nb_sup_valeur_sanitaire FROM - {{ ref('int__resultats_cvm_udi_dernier') }} + {{ ref('int__resultats_sub_indus_udi_dernier') }} UNION ALL +-- nitrites SELECT cdreseau, periode, @@ -98,4 +99,3 @@ SELECT nb_parametres FROM {{ ref('int__resultats_nitrites_udi_dernier') }} - From 550edba41407b480da5cc9a820d2800ac2fd3528 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Thu, 17 Apr 2025 18:35:29 +0200 Subject: [PATCH 20/30] fix --- dbt_/models/intermediate/int__union_resultats_udi.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index 42b496b2..48c33506 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -95,7 +95,9 @@ SELECT resultat, null AS ratio, dernier_prel_datetime, - null AS dernier_prel_valeur, - nb_parametres + dernier_prel_valeur, + nb_parametres, + null AS nb_prelevements, + null AS nb_sup_valeur_sanitaire FROM {{ ref('int__resultats_nitrites_udi_dernier') }} From c8bfca5add241bfa4e27f20e72ebb75fdc080f6b Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Thu, 17 Apr 2025 18:43:13 +0200 Subject: [PATCH 21/30] fix --- .../intermediate/int__union_resultats_udi.sql | 13 +++++++++---- .../nitrites/int__resultats_nitrites_udi_annuel.sql | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index e1735978..ebfb2e0b 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -87,6 +87,7 @@ SELECT FROM {{ ref('int__resultats_sub_indus_udi_dernier') }} UNION ALL +-- nitrites SELECT cdreseau, periode, @@ -94,8 +95,10 @@ SELECT resultat, null AS ratio, dernier_prel_datetime, - null AS dernier_prel_valeur, - nb_parametres + dernier_prel_valeur, + nb_parametres, + null AS nb_prelevements, + null AS nb_sup_valeur_sanitaire FROM {{ ref('int__resultats_nitrites_udi_dernier') }} UNION ALL @@ -104,9 +107,11 @@ SELECT periode, categorie, null AS resultat, - ratio_depassements AS ratio, + ratio_limite_sanitaire AS ratio, null AS dernier_prel_datetime, null AS dernier_prel_valeur, - null AS nb_parametres + null AS nb_parametres, + nb_prelevements, + null AS nb_sup_valeur_sanitaire FROM {{ ref('int__resultats_nitrites_udi_annuel') }} diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql index 71c883ca..c0f03487 100644 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql +++ b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql @@ -105,7 +105,7 @@ SELECT )::float / COUNT(DISTINCT prels.referenceprel)::float - ) AS ratio_depassements + ) AS ratio_limite_sanitaire FROM prels CROSS JOIN valeur_ref From cfdb0c838de794e7439bad06a55f227930e09c4a Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Thu, 17 Apr 2025 18:50:56 +0200 Subject: [PATCH 22/30] fix --- dbt_/models/intermediate/int__union_resultats_udi.sql | 2 +- dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index 48c33506..089a5f12 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -95,7 +95,7 @@ SELECT resultat, null AS ratio, dernier_prel_datetime, - dernier_prel_valeur, + null AS dernier_prel_valeur, nb_parametres, null AS nb_prelevements, null AS nb_sup_valeur_sanitaire diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index 1e35efe6..280f6f45 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -30,9 +30,6 @@ models: - not_null - dbt_utils.expression_is_true: expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" - - name: dernier_prel_valeur - description: "valeur mesurée sur le dernier prélèvement considéré" - type: FLOAT - name: nb_parametres description: "nombre de paramètres mesurés sur le prélèvement pour la catégorie considérée" type: INT From b52ddeccd3e13fc96595a1f0ceb871f3d3bc1e08 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Thu, 17 Apr 2025 18:51:59 +0200 Subject: [PATCH 23/30] fix --- dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index b7c06f09..f5f4c288 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -30,9 +30,6 @@ models: - not_null - dbt_utils.expression_is_true: expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" - - name: dernier_prel_valeur - description: "valeur mesurée sur le dernier prélèvement considéré" - type: FLOAT - name: nb_parametres description: "nombre de paramètres mesurés sur le prélèvement pour la catégorie considérée" type: INT @@ -77,7 +74,7 @@ models: type: INT - name: nb_prelevements type: INT - - name: ratio_depassements + - name: ratio_limite_sanitaire description: "Pourcentage de prélèvements qui dépassent la limite" type: FLOAT tests: From f59b3721d2f6fffb163399be1bdaee6e25f879f6 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 18 Apr 2025 10:15:50 +0200 Subject: [PATCH 24/30] fix: ratio_limite_sanitaire into test --- dbt_/tests/test_nitrates_results.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index fae8dc1d..02058c2e 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -74,7 +74,7 @@ SELECT '' AS resultat, nb_depassements, nb_prelevements, - ratio_depassements + ratio_limite_sanitaire FROM {{ ref('int__resultats_nitrites_udi_annuel') }} WHERE From bdd9718e02218bf72781a954ac59ed347c748374 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 18 Apr 2025 10:17:52 +0200 Subject: [PATCH 25/30] fix: typo into yaml --- dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml index f5f4c288..d59c5502 100644 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml @@ -42,7 +42,7 @@ models: - accepted_values: values: - inf_limite_qualite - - sup_limite_qualit + - sup_limite_qualite - name: int__resultats_nitrites_udi_annuel description: Résultats nitrites par UDI pour la temporalité 'bilan_annuel' From 45ffbf654519d7cc15f97dd3329d326baaa6144f Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:29:59 +0200 Subject: [PATCH 26/30] fix: conflit and rename nitrites to nitrate --- .../intermediate/int__union_resultats_udi.sql | 4 +- .../nitrate/_int__nitrate_models.yaml | 41 +++++ .../int__resultats_nitrate_udi_annuel.sql} | 0 .../nitrites/_int__nitrites_models.yaml | 84 --------- .../int__resultats_nitrites_udi_dernier.sql | 174 ------------------ dbt_/tests/test_nitrates_results.sql | 2 +- 6 files changed, 44 insertions(+), 261 deletions(-) rename dbt_/models/intermediate/{nitrites/int__resultats_nitrites_udi_annuel.sql => nitrate/int__resultats_nitrate_udi_annuel.sql} (100%) delete mode 100644 dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml delete mode 100644 dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index 203747ed..d1833238 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -115,7 +115,7 @@ SELECT null AS nb_prelevements, null AS nb_sup_valeur_sanitaire FROM - {{ ref('int__resultats_nitrites_udi_dernier') }} + {{ ref('int__resultats_nitrate_udi_dernier') }} UNION ALL SELECT cdreseau, @@ -129,4 +129,4 @@ SELECT nb_prelevements, null AS nb_sup_valeur_sanitaire FROM - {{ ref('int__resultats_nitrites_udi_annuel') }} + {{ ref('int__resultats_nitrate_udi_annuel') }} diff --git a/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml b/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml index e56cb009..78eac01e 100644 --- a/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml +++ b/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml @@ -45,3 +45,44 @@ models: values: - inf_limite_qualite - sup_limite_qualite + + - name: int__resultats_nitrate_udi_annuel + description: Résultats nitrates par UDI pour la temporalité 'bilan_annuel' + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cdreseau + - annee + columns: + - name: cdreseau + description: "Code de l'installation (unité de distribution)" + tests: + - not_null + - name: annee + description: "Annee du prevelvement" + type: SMALLINT + - name: categorie + description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + tests: + - accepted_values: + values: ["nitrate"] + - name: periode + description: "Période du résultat" + type: VARCHAR + tests: + - dbt_utils.expression_is_true: + expression: "LIKE 'bilan_annuel%'" + - name: nb_depassements + description: "Nombre de prélèvements qui dépassent la valeur de référence" + type: INTEGER + - name: nb_prelevements + description: "Nombre de prélèvements total" + type: INTEGER + - name: ratio_limite_sanitaire + description: "Pourcentage de prélèvements qui dépassent la limite" + type: FLOAT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql b/dbt_/models/intermediate/nitrate/int__resultats_nitrate_udi_annuel.sql similarity index 100% rename from dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_annuel.sql rename to dbt_/models/intermediate/nitrate/int__resultats_nitrate_udi_annuel.sql diff --git a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml b/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml deleted file mode 100644 index d59c5502..00000000 --- a/dbt_/models/intermediate/nitrites/_int__nitrites_models.yaml +++ /dev/null @@ -1,84 +0,0 @@ -version: 2 - -models: - - name: int__resultats_nitrites_udi_dernier - description: Resultats nitrites par UDI pour la temporalité 'derniers prélèvements' - columns: - - name: cdreseau - description: "Code de l'installation (unité de distribution)" - type: VARCHAR - tests: - - not_null - - unique - - name: categorie - description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." - type: VARCHAR - tests: - - not_null - - accepted_values: - values: ["nitrate"] - - name: periode - description: "Période du résultat" - type: VARCHAR - tests: - - accepted_values: - values: ["dernier_prel"] - - name: dernier_prel_datetime - description: "Date du dernier prélèvement considéré" - type: TIMESTAMP - tests: - - not_null - - dbt_utils.expression_is_true: - expression: ">= CURRENT_DATE - INTERVAL 1 YEAR" - - name: nb_parametres - description: "nombre de paramètres mesurés sur le prélèvement pour la catégorie considérée" - type: INT - tests: - - not_null - - name: resultat - description: "classification de la substance pour le prélèvement" - tests: - - not_null - - accepted_values: - values: - - inf_limite_qualite - - sup_limite_qualite - - - name: int__resultats_nitrites_udi_annuel - description: Résultats nitrites par UDI pour la temporalité 'bilan_annuel' - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - cdreseau - - annee - columns: - - name: cdreseau - description: "Code de l'installation (unité de distribution)" - tests: - - not_null - - name: annee - description: "Annee du prevelvement" - type: SMALLINT - - name: categorie - description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." - tests: - - accepted_values: - values: ["nitrate"] - - name: periode - description: "Période du résultat" - type: VARCHAR - tests: - - dbt_utils.expression_is_true: - expression: "LIKE 'bilan_annuel%'" - - name: nb_depassements - type: INT - - name: nb_prelevements - type: INT - - name: ratio_limite_sanitaire - description: "Pourcentage de prélèvements qui dépassent la limite" - type: FLOAT - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - max_value: 1 diff --git a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql b/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql deleted file mode 100644 index baf00075..00000000 --- a/dbt_/models/intermediate/nitrites/int__resultats_nitrites_udi_dernier.sql +++ /dev/null @@ -1,174 +0,0 @@ -WITH -last_pvl AS ( - SELECT - cdreseau, - referenceprel, - datetimeprel, - ROW_NUMBER() - OVER ( - PARTITION BY cdreseau, cdparametresiseeaux - ORDER BY datetimeprel DESC - ) - AS row_number - FROM - {{ ref('int__resultats_udi_communes') }} - WHERE - categorie = 'nitrate' - AND cdparametresiseeaux = 'NO3' - AND -- On garde les prélèvements de moins d'un an - CURRENT_DATE - datetimeprel < INTERVAL 1 YEAR - AND valtraduite IS NOT NULL - -- Cf cas cdreseau IN( '034005906') , referenceprel= 03400327764 -), - -valeur_ref AS ( - SELECT - MAX( - CASE - WHEN - cdparametresiseeaux = 'NO3' - THEN limite_qualite - END - ) AS limite_qualite_no3, - MAX( - CASE - WHEN - cdparametresiseeaux = 'NO3_NO2' - THEN limite_qualite - END - ) AS limite_qualite_no3_no2, - MAX( - CASE - WHEN - cdparametresiseeaux = 'NO2' - THEN limite_qualite - END - ) AS limite_qualite_no2 - FROM - {{ ref('int__valeurs_de_reference') }} - WHERE - categorie_1 = 'nitrate' -), - -split_nitrites AS ( - SELECT - resultats.cdreseau, - resultats.categorie, - resultats.datetimeprel AS dernier_prel_datetime, - COUNT(DISTINCT resultats.cdparametresiseeaux) AS nb_parametres, - MAX( - CASE - WHEN - resultats.cdparametresiseeaux = 'NO3' - THEN resultats.valtraduite - END - ) AS valtraduite_no3, - MAX( - CASE - WHEN - resultats.cdparametresiseeaux = 'NO2' - THEN resultats.valtraduite - END - ) AS valtraduite_no2, - MAX( - CASE - WHEN - resultats.cdparametresiseeaux = 'NO3_NO2' - THEN resultats.valtraduite - END - ) AS valtraduite_no3_no2 - FROM - last_pvl - LEFT JOIN - {{ ref('int__resultats_udi_communes') }} AS resultats - ON - last_pvl.cdreseau = resultats.cdreseau - AND last_pvl.referenceprel = resultats.referenceprel - WHERE - last_pvl.row_number = 1 - -- on ne veut que le dernier prélèvement - AND resultats.categorie = 'nitrate' - -- on ne veut que les résultats associé aux nitrates/nitrites - GROUP BY - resultats.cdreseau, - resultats.categorie, - resultats.datetimeprel -), - -split_nitrites_with_ref AS ( - SELECT - split_nitrites.cdreseau, - split_nitrites.categorie, - split_nitrites.nb_parametres, - split_nitrites.dernier_prel_datetime, - valeur_ref.limite_qualite_no3, - valeur_ref.limite_qualite_no2, - valeur_ref.limite_qualite_no3_no2, - split_nitrites.valtraduite_no3_no2, - split_nitrites.valtraduite_no3, - split_nitrites.valtraduite_no2 - FROM - split_nitrites - CROSS JOIN - valeur_ref -) - -SELECT - split_nitrites_with_ref.cdreseau, - split_nitrites_with_ref.categorie, - 'dernier_prel' AS periode, - split_nitrites_with_ref.dernier_prel_datetime, - split_nitrites_with_ref.nb_parametres, - CASE - WHEN -- Si nitrates (no3) et pas nitrites (no2) - split_nitrites_with_ref.valtraduite_no3 IS NOT NULL - AND split_nitrites_with_ref.valtraduite_no2 IS NULL - AND - split_nitrites_with_ref.valtraduite_no3 - < split_nitrites_with_ref.limite_qualite_no3 - THEN 'inf_limite_qualite' - WHEN - split_nitrites_with_ref.valtraduite_no3 IS NOT NULL - AND split_nitrites_with_ref.valtraduite_no2 IS NULL - AND - split_nitrites_with_ref.valtraduite_no3 - >= split_nitrites_with_ref.limite_qualite_no3 - THEN 'sup_limite_qualite' - WHEN -- Si nitrates (no3) ET nitrites (no2) - split_nitrites_with_ref.valtraduite_no3 IS NOT NULL - AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL - AND ( - split_nitrites_with_ref.valtraduite_no3 - >= split_nitrites_with_ref.limite_qualite_no3 - OR split_nitrites_with_ref.valtraduite_no2 - >= split_nitrites_with_ref.limite_qualite_no2 - OR COALESCE( - split_nitrites_with_ref.valtraduite_no3_no2, - ( - split_nitrites_with_ref.valtraduite_no3 / 50 - + split_nitrites_with_ref.valtraduite_no2 / 3 - ) - ) - >= split_nitrites_with_ref.limite_qualite_no3_no2 - ) - THEN 'sup_limite_qualite' - WHEN - split_nitrites_with_ref.valtraduite_no3 IS NOT NULL - AND split_nitrites_with_ref.valtraduite_no2 IS NOT NULL - AND split_nitrites_with_ref.valtraduite_no3 - < split_nitrites_with_ref.limite_qualite_no3 - AND split_nitrites_with_ref.valtraduite_no2 - < split_nitrites_with_ref.limite_qualite_no2 - AND COALESCE( - split_nitrites_with_ref.valtraduite_no3_no2, - ( - split_nitrites_with_ref.valtraduite_no3 / 50 - + split_nitrites_with_ref.valtraduite_no2 / 3 - ) - ) - < split_nitrites_with_ref.limite_qualite_no3_no2 - THEN 'inf_limite_qualite' - ELSE 'erreur' - END AS resultat -FROM - split_nitrites_with_ref diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 2a1f87bc..dc4cd9c8 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -61,7 +61,7 @@ WHERE cdreseau = '011000196' AND dernier_prel_datetime = '2024-11-27 10:57:00' AND resultat != 'inf_limite_qualite' - ) + ) UNION ALL -- annuel SELECT From 5165df0479fc965e8a8b72f68e331ecff4b23a36 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:33:37 +0200 Subject: [PATCH 27/30] fix: add dernier_prel_valeur into int__union_resultats_udi for nitrate_dernier --- dbt_/models/intermediate/int__union_resultats_udi.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index d1833238..339f3bad 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -110,7 +110,7 @@ SELECT resultat, null AS ratio, dernier_prel_datetime, - null AS dernier_prel_valeur, + dernier_prel_valeur, nb_parametres, null AS nb_prelevements, null AS nb_sup_valeur_sanitaire From 916c9b7e00c6a1e9667d78f771dc87afaacca048 Mon Sep 17 00:00:00 2001 From: Vinca PRANA <11522296+20k-P@users.noreply.github.com> Date: Fri, 25 Apr 2025 13:50:24 +0200 Subject: [PATCH 28/30] fix: test update --- dbt_/tests/test_nitrates_results.sql | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index dc4cd9c8..07755064 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -67,17 +67,15 @@ UNION ALL SELECT 'annuel' AS periode, cdreseau, - categorie, '' AS resultat, nb_depassements, nb_prelevements, ratio_limite_sanitaire FROM - {{ ref('int__resultats_nitrites_udi_annuel') }} + {{ ref('int__resultats_nitrate_udi_annuel') }} WHERE ( cdreseau = '092003070' - AND categorie = 'nitrate' AND annee = '2024' AND ( nb_prelevements != 806 @@ -87,7 +85,6 @@ WHERE OR ( cdreseau = '071001155' - AND categorie = 'nitrate' AND annee = '2023' AND ( nb_prelevements != 1 @@ -97,7 +94,6 @@ WHERE OR ( cdreseau = '036000670' - AND categorie = 'nitrate' AND annee = '2024' AND ( nb_prelevements != 27 @@ -107,14 +103,12 @@ WHERE OR ( cdreseau = '092003070' - AND categorie = 'nitrate' AND annee IN (2020, 2021, 2022, 2023, 2024, 2025) AND nb_depassements != 0 ) OR ( cdreseau = '089003503' - AND categorie = 'nitrate' AND annee = '2020' AND ( nb_prelevements != 12 From ded5d38c54146f0c4531e69629e28680d548b95d Mon Sep 17 00:00:00 2001 From: Jeremy Greze Date: Thu, 22 May 2025 13:42:27 +0200 Subject: [PATCH 29/30] misc --- .../intermediate/int__union_resultats_udi.sql | 58 ++++++------- .../nitrate/_int__nitrate_models.yaml | 84 +++++++++---------- .../int__resultats_nitrate_udi_annuel.sql | 13 ++- dbt_/tests/test_nitrates_results.sql | 2 +- 4 files changed, 77 insertions(+), 80 deletions(-) diff --git a/dbt_/models/intermediate/int__union_resultats_udi.sql b/dbt_/models/intermediate/int__union_resultats_udi.sql index 339f3bad..d2da0018 100644 --- a/dbt_/models/intermediate/int__union_resultats_udi.sql +++ b/dbt_/models/intermediate/int__union_resultats_udi.sql @@ -29,6 +29,35 @@ SELECT FROM {{ ref('int__resultats_cvm_udi_dernier') }} UNION ALL +-- nitrate +SELECT + cdreseau, + periode, + categorie, + null AS resultat, + ratio, + null AS dernier_prel_datetime, + null AS dernier_prel_valeur, + null AS nb_parametres, + nb_prelevements, + null AS nb_sup_valeur_sanitaire +FROM + {{ ref('int__resultats_nitrate_udi_annuel') }} +UNION ALL +SELECT + cdreseau, + periode, + categorie, + resultat, + null AS ratio, + dernier_prel_datetime, + dernier_prel_valeur, + nb_parametres, + null AS nb_prelevements, + null AS nb_sup_valeur_sanitaire +FROM + {{ ref('int__resultats_nitrate_udi_dernier') }} +UNION ALL -- pesticide/metabolite SELECT cdreseau, @@ -101,32 +130,3 @@ SELECT null AS nb_sup_valeur_sanitaire FROM {{ ref('int__resultats_sub_indus_udi_dernier') }} -UNION ALL --- nitrate -SELECT - cdreseau, - periode, - categorie, - resultat, - null AS ratio, - dernier_prel_datetime, - dernier_prel_valeur, - nb_parametres, - null AS nb_prelevements, - null AS nb_sup_valeur_sanitaire -FROM - {{ ref('int__resultats_nitrate_udi_dernier') }} -UNION ALL -SELECT - cdreseau, - periode, - categorie, - null AS resultat, - ratio_limite_sanitaire AS ratio, - null AS dernier_prel_datetime, - null AS dernier_prel_valeur, - null AS nb_parametres, - nb_prelevements, - null AS nb_sup_valeur_sanitaire -FROM - {{ ref('int__resultats_nitrate_udi_annuel') }} diff --git a/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml b/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml index 78eac01e..31fc911c 100644 --- a/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml +++ b/dbt_/models/intermediate/nitrate/_int__nitrate_models.yaml @@ -1,6 +1,47 @@ version: 2 models: + - name: int__resultats_nitrate_udi_annuel + description: Résultats nitrates par UDI pour la temporalité 'bilan_annuel' + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cdreseau + - annee + columns: + - name: cdreseau + description: "Code de l'installation (unité de distribution)" + tests: + - not_null + - name: annee + description: "Annee du prevelvement" + type: SMALLINT + - name: categorie + description: "Catégorie du paramètre (pesticides, pfas, cvm, nitrates, etc.) selon la classification établie." + tests: + - accepted_values: + values: ["nitrate"] + - name: periode + description: "Période du résultat" + type: VARCHAR + tests: + - dbt_utils.expression_is_true: + expression: "LIKE 'bilan_annuel%'" + - name: nb_depassements + description: "Nombre de prélèvements qui dépassent la valeur de référence" + type: INTEGER + - name: nb_prelevements + description: "Nombre de prélèvements total" + type: INTEGER + - name: ratio + description: "Pourcentage de prélèvements qui dépassent la limite" + type: FLOAT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1 + - name: int__resultats_nitrate_udi_dernier description: Resultats nitrate par UDI pour la temporalité 'derniers prélèvements' columns: @@ -11,7 +52,7 @@ models: - not_null - unique - name: categorie - description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." + description: "Catégorie du paramètre (pesticides, pfas, cvm, nitrates, etc.) selon la classification établie." type: VARCHAR tests: - not_null @@ -45,44 +86,3 @@ models: values: - inf_limite_qualite - sup_limite_qualite - - - name: int__resultats_nitrate_udi_annuel - description: Résultats nitrates par UDI pour la temporalité 'bilan_annuel' - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - cdreseau - - annee - columns: - - name: cdreseau - description: "Code de l'installation (unité de distribution)" - tests: - - not_null - - name: annee - description: "Annee du prevelvement" - type: SMALLINT - - name: categorie - description: "Catégorie du paramètre (pesticides, pfas, vcm, nitrates, etc.) selon la classification établie." - tests: - - accepted_values: - values: ["nitrate"] - - name: periode - description: "Période du résultat" - type: VARCHAR - tests: - - dbt_utils.expression_is_true: - expression: "LIKE 'bilan_annuel%'" - - name: nb_depassements - description: "Nombre de prélèvements qui dépassent la valeur de référence" - type: INTEGER - - name: nb_prelevements - description: "Nombre de prélèvements total" - type: INTEGER - - name: ratio_limite_sanitaire - description: "Pourcentage de prélèvements qui dépassent la limite" - type: FLOAT - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_between: - min_value: 0 - max_value: 1 diff --git a/dbt_/models/intermediate/nitrate/int__resultats_nitrate_udi_annuel.sql b/dbt_/models/intermediate/nitrate/int__resultats_nitrate_udi_annuel.sql index c0f03487..c592702c 100644 --- a/dbt_/models/intermediate/nitrate/int__resultats_nitrate_udi_annuel.sql +++ b/dbt_/models/intermediate/nitrate/int__resultats_nitrate_udi_annuel.sql @@ -4,7 +4,7 @@ prels AS ( de_partition AS annee, cdreseau, referenceprel, - datetimeprel, + MAX(datetimeprel) AS datetimeprel, COUNT(DISTINCT cdparametresiseeaux) AS nb_parametres, MAX( CASE @@ -37,8 +37,7 @@ prels AS ( GROUP BY annee, cdreseau, - referenceprel, - datetimeprel, + referenceprel ), valeur_ref AS ( @@ -81,8 +80,7 @@ SELECT WHEN prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3 OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2 - OR prels.valtraduite_no3_no2 - >= valeur_ref.limite_qualite_no3_no2 + OR prels.valtraduite_no3_no2 >= valeur_ref.limite_qualite_no3_no2 OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3 >= valeur_ref.limite_qualite_no3_no2 THEN prels.referenceprel @@ -96,8 +94,7 @@ SELECT WHEN prels.valtraduite_no3 >= valeur_ref.limite_qualite_no3 OR prels.valtraduite_no2 >= valeur_ref.limite_qualite_no2 - OR prels.valtraduite_no3_no2 - >= valeur_ref.limite_qualite_no3_no2 + OR prels.valtraduite_no3_no2 >= valeur_ref.limite_qualite_no3_no2 OR prels.valtraduite_no3 / 50 + prels.valtraduite_no2 / 3 >= valeur_ref.limite_qualite_no3_no2 THEN prels.referenceprel @@ -105,7 +102,7 @@ SELECT )::float / COUNT(DISTINCT prels.referenceprel)::float - ) AS ratio_limite_sanitaire + ) AS ratio FROM prels CROSS JOIN valeur_ref diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 07755064..5a8a5cec 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -70,7 +70,7 @@ SELECT '' AS resultat, nb_depassements, nb_prelevements, - ratio_limite_sanitaire + ratio FROM {{ ref('int__resultats_nitrate_udi_annuel') }} WHERE From a3356c5112a9e0c8bb8344c40886a56a821c0c1d Mon Sep 17 00:00:00 2001 From: Jeremy Greze Date: Thu, 22 May 2025 14:44:43 +0200 Subject: [PATCH 30/30] update tests --- dbt_/tests/test_nitrates_results.sql | 31 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/dbt_/tests/test_nitrates_results.sql b/dbt_/tests/test_nitrates_results.sql index 5a8a5cec..4206fcbb 100644 --- a/dbt_/tests/test_nitrates_results.sql +++ b/dbt_/tests/test_nitrates_results.sql @@ -80,6 +80,7 @@ WHERE AND ( nb_prelevements != 806 OR nb_depassements != 0 + OR ratio != 0 ) ) OR @@ -97,21 +98,35 @@ WHERE AND annee = '2024' AND ( nb_prelevements != 27 - OR nb_depassements != 27 + OR nb_depassements != 27 -- depassements de N03 ) ) OR - ( - cdreseau = '092003070' - AND annee IN (2020, 2021, 2022, 2023, 2024, 2025) - AND nb_depassements != 0 - ) - OR ( cdreseau = '089003503' AND annee = '2020' AND ( - nb_prelevements != 12 + nb_prelevements != 12 -- depassements de N03 et N03_N02 OR nb_depassements != 3 + OR ratio != 0.25 + ) + ) + OR + ( + cdreseau = '055000713' + AND annee = '2023' + AND ( + nb_prelevements != 14 + OR nb_depassements != 6 -- depassements de N02 + OR ratio < 0.42 + ) + ) + OR + ( + cdreseau = '027000943' + AND annee = '2021' + AND ( + nb_prelevements != 63 + OR nb_depassements != 4 -- depassements de N03 et N03_N02 ) )