diff --git a/dags/miovision_check.py b/dags/miovision_check.py index 6a6a33f95..dfbf3964b 100644 --- a/dags/miovision_check.py +++ b/dags/miovision_check.py @@ -95,10 +95,20 @@ def miovision_check_dag(): Identify high volume movements missing from intersection_movements table and notify. ''' + check_missing_centreline_ids = SQLCheckOperatorWithReturnValue( + task_id="check_missing_centreline_ids", + sql="select-missing-centreline_ids.sql", + conn_id="miovision_api_bot" + ) + check_monitor_intersection_movements.doc_md = ''' + Identify centreline_ids missing/outdated in centreline_miovision table and notify. + ''' + t_upstream_done >> [ check_distinct_intersection_uid, check_open_anomalous_ranges, - check_monitor_intersection_movements + check_monitor_intersection_movements, + check_missing_centreline_ids ] miovision_check_dag() \ No newline at end of file diff --git a/volumes/miovision/sql/data_checks/select-missing-centreline_ids.sql b/volumes/miovision/sql/data_checks/select-missing-centreline_ids.sql new file mode 100644 index 000000000..33d6b8757 --- /dev/null +++ b/volumes/miovision/sql/data_checks/select-missing-centreline_ids.sql @@ -0,0 +1,60 @@ +/* +This query looks for intersection legs with no entry or outdated entry in `centreline_miovision`. + +Exceptions: +- Leg is labelled as restricted in intersections table +- When leg does not exist in centreline, but is a valid movement, add a null entry in `centreline_miovision` to opt out of notifications. +*/ + +WITH valid_legs AS ( + SELECT + intersection_uid, + UNNEST(ARRAY[ + CASE WHEN e_leg_restricted IS NULL THEN 'E' END, + CASE WHEN n_leg_restricted IS NULL THEN 'N' END, + CASE WHEN s_leg_restricted IS NULL THEN 'S' END, + CASE WHEN w_leg_restricted IS NULL THEN 'W' END + ]) AS leg + FROM miovision_api.active_intersections +), + +missing AS ( + SELECT + vl.intersection_uid, + ai.api_name, + ai.geom, + vl.leg, + CASE + WHEN + cl.centreline_id IS NULL + THEN 'Entry missing from `miovision_api.centreline_miovision`.' + WHEN + latest.centreline_id IS NULL + THEN 'Entry is outdated (no longer in `gis_core.centreline_latest`).' + END AS description + FROM valid_legs AS vl + JOIN miovision_api.active_intersections AS ai USING (intersection_uid) + LEFT JOIN miovision_api.centreline_miovision AS cl USING (intersection_uid, leg) + LEFT JOIN gis_core.centreline_latest AS latest USING (centreline_id) + WHERE + vl.leg IS NOT NULL + AND ( + cl.intersection_uid IS NULL --not in table (allowing for nulls) + --entry exists, but is no longer valid + OR (cl.centreline_id IS NOT NULL AND latest.centreline_id IS NULL) + ) + ORDER BY intersection_uid +) + +SELECT + NOT(COUNT(*) > 0) AS _check, + CASE WHEN COUNT(*) = 1 THEN 'There is ' ELSE 'There are ' END || COUNT(*) + || ' Miovision legs which are missing/outdated in `miovision_api.centreline_miovision`. ' + || 'See [readme](https://github.com/CityofToronto/bdit_data-sources/tree/master/volumes/miovision/sql/#centreline_miovision).' --noqa + AS summ, + array_agg( + 'intersection_uid: `' || intersection_uid + || '`, leg: `' || leg || '` ' + || description + ) AS gaps +FROM missing diff --git a/volumes/miovision/sql/readme.md b/volumes/miovision/sql/readme.md index 2191fbbe4..3eae74e39 100644 --- a/volumes/miovision/sql/readme.md +++ b/volumes/miovision/sql/readme.md @@ -466,7 +466,10 @@ leg| text | A segment that forms part of a miovision intersection, identified by **Known issues:** - 91: Lakeshore and Spadina - Two centrelines matched to West leg are actually legit. One is for the Gardiner off-ramp (Left turns only) and one is for Lakeshore (Thru + Right). They could be differentiated by movement. - 78: Bloor and Kingsway is a 4 legged intersection, but the south leg is not in the centreline (private road / cemetery entrance). -- 68: Steeles and Jane, N leg is outside of TO. +- 68: Steeles and Jane, N leg is outside of TO (used null entry). +- 105: Bayview and Brickworks - North leg (entry into Brickworks) is not on centreline (used null entry). +- 125: Two north legs (highway ramps) +- 123,124,127: Queensway midblock cameras; legs would not make sense. ### `alerts`