Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
f58f84c
Use `airflow.sdk.get_current_context` instead #1211
gabrielwol May 5, 2025
9a33c3f
#1211 replace outdated template references
gabrielwol May 5, 2025
81e448f
switch pythonoperator and bashoperator to apache-airflow-providers-st…
gabrielwol May 5, 2025
0bea95f
#1211 commit to data-sources and fix up 3 here/bt agg DAGs
gabrielwol May 6, 2025
19d7917
delete 5 deprecated DAGs #1211
gabrielwol May 6, 2025
3c7d88b
#1211 ShortCircuitOperator -> task.short_circuit
gabrielwol May 6, 2025
13ecd67
#1211 add congestion_temp uncommited DAG
gabrielwol May 6, 2025
e1608fc
#1211 taskflow traffic_signals_dag
gabrielwol May 6, 2025
a9b247b
#1211 fix taskflow of traffic_signals_dag
gabrielwol May 7, 2025
153f4f9
#1211 update airflow constraints to 3.0.0, pip-compile requirements
gabrielwol May 7, 2025
775b583
#1211 fluff
gabrielwol May 7, 2025
9af8c94
#1211 rename some DAGs file to match dag_id
gabrielwol May 7, 2025
b8ba466
commit to data-sources and fix up 3 here/bt agg DAGs #1211
gabrielwol May 6, 2025
466a294
#1211 add congestion_temp uncommited DAG
gabrielwol May 6, 2025
a6fcb25
#1211 rename some DAGs file to match dag_id
gabrielwol May 7, 2025
a843f95
#1211 make start_date timezone aware w/pendulum
gabrielwol May 16, 2025
f7877cb
change imports #1211
gabrielwol May 23, 2025
d4eea54
#1211 temp airflow 2 fix
gabrielwol May 27, 2025
03fb2fd
#1211 update dag_functions path
gabrielwol May 27, 2025
acf88df
#1211 temp changes to queens_park_refresh
gabrielwol Jul 7, 2025
1737123
#1211 update airflow constraints to 3.0.2, remove apache-airflow-prov…
gabrielwol Jul 7, 2025
91cda23
move dag_owners to local file #1211
gabrielwol May 28, 2025
1f6dedc
update bdit_dag_utils commit #1211
gabrielwol Jul 11, 2025
f7e5bc5
#1211 forgot use_proxy in vds dags
gabrielwol Jul 22, 2025
0e0a203
move Variable import to airflow.sdk #1211
gabrielwol Jul 30, 2025
b6a0f6a
#1253 fix vds_check imports
gabrielwol Aug 6, 2025
814f8e1
#1211 dag owners chris -> david
gabrielwol Sep 22, 2025
057056d
#1211 address Airflow 3.1.0 deprecation warnings
gabrielwol Sep 30, 2025
7331755
#1211 switch vds to triggerdagrunoperator from externaltaskmarker/sensor
gabrielwol Oct 1, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bdit_dag_utils
251 changes: 94 additions & 157 deletions constraints.txt
Original file line number Diff line number Diff line change
@@ -1,193 +1,130 @@

#
# This constraints file was automatically generated on 2024-09-16T16:21:46.279414
# via "eager-upgrade" mechanism of PIP. For the "v2-10-test" branch of Airflow.
# This constraints file was automatically generated on 2025-06-03T13:56:17.557640
# via `uv sync --resolution highest` for the "v3-0-test" branch of Airflow.
# This variant of constraints install just the 'bare' 'apache-airflow' package build from the HEAD of
# the branch, without installing any of the providers.
#
# Those constraints represent the "newest" dependencies airflow could use, if providers did not limit
# Airflow in any way.
#
Authlib==1.3.2
ConfigUpdater==3.2
Deprecated==1.2.14
Flask-AppBuilder==4.5.0
Flask-Babel==2.0.0
Flask-Bcrypt==1.0.1
Flask-Caching==2.3.0
Flask-JWT-Extended==4.6.0
Flask-Limiter==3.8.0
Flask-Login==0.6.3
Flask-SQLAlchemy==2.5.1
Flask-Session==0.5.0
Flask-WTF==1.2.1
Deprecated==1.2.18
Flask==2.2.5
Jinja2==3.1.4
Mako==1.3.5
MarkupSafe==2.1.5
PyJWT==2.9.0
Jinja2==3.1.6
Mako==1.3.10
MarkupSafe==3.0.2
PyJWT==2.10.1
PyYAML==6.0.2
Pygments==2.18.0
Pygments==2.19.1
SQLAlchemy-JSONField==1.0.2
SQLAlchemy-Utils==0.41.2
SQLAlchemy==1.4.54
WTForms==3.1.2
Werkzeug==2.2.3
aiobotocore==2.15.0
aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aioitertools==0.12.0
aiosignal==1.3.1
alembic==1.13.2
amqp==5.2.0
a2wsgi==1.10.8
aiologic==0.14.0
aiosqlite==0.21.0
alembic==1.16.1
annotated-types==0.7.0
anyio==4.4.0
apispec==6.6.1
argcomplete==3.5.0
anyio==4.9.0
argcomplete==3.6.2
asgiref==3.8.1
async-timeout==4.0.3
atlasclient==1.0.0
attrs==24.2.0
babel==2.16.0
backoff==2.2.1
bcrypt==4.2.0
blinker==1.8.2
botocore==1.35.16
cachelib==0.9.0
certifi==2024.8.30
attrs==25.3.0
backports.strenum==1.3.1
cadwyn==5.3.3
certifi==2025.4.26
cffi==1.17.1
cgroupspy==0.2.3
charset-normalizer==3.3.2
click==8.1.7
clickclick==20.10.2
cloudpickle==3.0.0
colorama==0.4.6
colorlog==6.8.2
connexion==2.14.2
charset-normalizer==3.4.2
click==8.1.8
colorlog==6.9.0
cron-descriptor==1.4.5
croniter==3.0.3
cryptography==43.0.1
decorator==5.1.1
dill==0.3.8
distlib==0.3.8
dnspython==2.6.1
docopt==0.6.2
docutils==0.21.2
croniter==6.0.0
cryptography==42.0.8
dill==0.3.1.1
dnspython==2.7.0
email_validator==2.2.0
eventlet==0.37.0
exceptiongroup==1.2.2
fastavro==1.9.7
filelock==3.16.0
frozenlist==1.4.1
fsspec==2024.9.0
gevent==24.2.1
google-re2==1.1.20240702
googleapis-common-protos==1.65.0
graphviz==0.20.3
greenlet==3.1.0
grpcio==1.66.1
gssapi==1.8.3
exceptiongroup==1.3.0
fastapi-cli==0.0.7
fastapi==0.115.12
fsspec==2025.5.1
googleapis-common-protos==1.70.0
greenlet==3.2.2
grpcio==1.72.1
gunicorn==23.0.0
h11==0.14.0
hdfs==2.7.3
httpcore==1.0.5
httpx==0.27.2
h11==0.16.0
httpcore==1.0.9
httptools==0.6.4
httpx==0.27.0
idna==3.10
importlib_metadata==8.5.0
importlib_resources==6.4.5
inflection==0.5.1
isodate==0.6.1
importlib_metadata==8.4.0
itsdangerous==2.2.0
jmespath==1.0.1
jsonschema-specifications==2023.12.1
jsonschema==4.23.0
krb5==0.6.0
lazy-object-proxy==1.10.0
ldap3==2.9.1
limits==3.13.0
jsonschema-specifications==2025.4.1
jsonschema==4.24.0
lazy-object-proxy==1.11.0
libcst==1.8.0
linkify-it-py==2.0.3
lockfile==0.12.2
lxml==5.3.0
markdown-it-py==3.0.0
marshmallow-oneofschema==3.1.1
marshmallow-sqlalchemy==0.28.2
marshmallow==3.22.0
mdit-py-plugins==0.4.2
mdurl==0.1.2
methodtools==0.4.7
more-itertools==10.5.0
multidict==6.1.0
numpy==1.26.4
opentelemetry-api==1.16.0
opentelemetry-exporter-otlp-proto-grpc==1.16.0
opentelemetry-exporter-otlp-proto-http==1.16.0
opentelemetry-exporter-otlp==1.16.0
opentelemetry-exporter-prometheus==0.41b0
opentelemetry-proto==1.16.0
opentelemetry-sdk==1.16.0
opentelemetry-semantic-conventions==0.37b0
ordered-set==4.1.0
packaging==24.1
pandas==2.1.4
more-itertools==10.7.0
msgspec==0.19.0
opentelemetry-api==1.27.0
opentelemetry-exporter-otlp-proto-common==1.27.0
opentelemetry-exporter-otlp-proto-grpc==1.27.0
opentelemetry-exporter-otlp-proto-http==1.27.0
opentelemetry-exporter-otlp==1.27.0
opentelemetry-proto==1.27.0
opentelemetry-sdk==1.27.0
opentelemetry-semantic-conventions==0.48b0
packaging==24.2
pathspec==0.12.1
pendulum==3.0.0
platformdirs==4.3.3
pluggy==1.5.0
plyvel==1.5.1
prison==0.2.1
prometheus_client==0.20.0
protobuf==4.25.4
psutil==6.0.0
pure-sasl==0.6.2
pyasn1==0.6.1
pyasn1_modules==0.4.1
pendulum==3.1.0
pluggy==1.6.0
protobuf==4.25.8
psutil==7.0.0
pycparser==2.22
pydantic==2.9.1
pydantic_core==2.23.3
pykerberos==1.2.4
pyspnego==0.11.1
python-daemon==3.0.1
pydantic==2.11.5
pydantic_core==2.33.2
python-daemon==3.1.2
python-dateutil==2.9.0.post0
python-ldap==3.4.4
python-nvd3==0.16.0
python-dotenv==1.1.0
python-multipart==0.0.20
python-slugify==8.0.4
python3-saml==1.16.0
pytz==2024.2
referencing==0.35.1
requests-kerberos==0.15.0
requests-toolbelt==1.0.0
pytz==2025.2
referencing==0.36.2
requests==2.32.3
rfc3339-validator==0.1.4
rich-argparse==1.5.2
rich==13.8.1
rpds-py==0.20.0
s3fs==2024.9.0
sentry-sdk==2.14.0
setproctitle==1.3.3
six==1.16.0
retryhttp==1.3.3
rich-argparse==1.7.1
rich-toolkit==0.14.7
rich==13.9.4
rpds-py==0.25.1
setproctitle==1.3.6
shellingham==1.5.4
six==1.17.0
sniffio==1.3.1
sqlparse==0.5.1
statsd==4.0.1
sqlparse==0.5.3
starlette==0.46.2
structlog==25.4.0
svcs==25.1.0
tabulate==0.9.0
tenacity==9.0.0
termcolor==2.4.0
tenacity==9.1.2
termcolor==3.1.0
text-unidecode==1.3
thrift-sasl==0.4.3
thrift==0.20.0
time-machine==2.15.0
typing_extensions==4.12.2
tzdata==2024.1
tomli==2.2.1
typer==0.16.0
types-requests==2.32.0.20250602
typing-inspection==0.4.1
typing_extensions==4.13.2
tzdata==2025.2
uc-micro-py==1.0.3
unicodecsv==0.14.1
universal_pathlib==0.2.5
urllib3==2.2.3
uv==0.4.1
vine==5.1.0
virtualenv==20.26.4
wirerope==0.4.7
wrapt==1.16.0
xmlsec==1.3.14
yarl==1.11.1
zipp==3.20.2
zope.event==5.0
zope.interface==7.0.3
universal_pathlib==0.2.6
urllib3==2.4.0
uuid6==2024.7.10
uv==0.7.8
uvicorn==0.34.3
uvloop==0.21.0
watchfiles==1.0.5
websockets==14.2
wirerope==1.0.0
wrapt==1.17.2
zipp==3.22.0
3 changes: 2 additions & 1 deletion dags/.airflowignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
^common_tasks.py$
^custom_operators.py$
^dag_functions.py$
^dag_functions.py$
^dag_owners.py$
8 changes: 4 additions & 4 deletions dags/bluetooth_check_readers_temp.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
import pendulum
from datetime import timedelta

from airflow.decorators import dag, task
from airflow.models import Variable
from airflow.sdk import dag, task
from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
from airflow.providers.postgres.hooks.postgres import PostgresHook

Expand All @@ -14,6 +13,7 @@
repo_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
sys.path.insert(0, repo_path)
# pylint: disable=wrong-import-position
from dags.dag_owners import owners
from bdit_dag_utils.utils.dag_functions import task_fail_slack_alert, slack_alert_data_quality
from bdit_dag_utils.utils.custom_operators import SQLCheckOperatorWithReturnValue
# pylint: enable=import-error
Expand All @@ -25,7 +25,7 @@
logging.basicConfig(level=logging.DEBUG)

DAG_NAME = 'temp_bluetooth_check_readers'
DAG_OWNERS = Variable.get("dag_owners", deserialize_json=True).get(DAG_NAME, ["Unknown"])
DAG_OWNERS = owners.get(DAG_NAME, ["Unknown"])

def format_br_list(returned_list):
# Format broken reader list into a text for slack message.
Expand Down Expand Up @@ -63,7 +63,7 @@ def blip_pipeline():
sql = '''SELECT (COUNT(*) > 0) AS "_check",
'There are no data inserted for '|| '{{ ds }}' AS msg
FROM bluetooth.aggr_5min
WHERE datetime_bin >='{{ ds }}' and datetime_bin < '{{ tomorrow_ds }}'
WHERE datetime_bin >='{{ ds }}' and datetime_bin < '{{ macros.ds_add(ds, 1) }}'
LIMIT 1'''
)

Expand Down
Loading