feat(5811): update Airflow DAGs failure callback

junminahn · junminahn · commit f6b651958eaa · 2025-06-13T14:18:54.000-07:00
diff --git a/Makefile b/Makefile
@@ -32,6 +32,8 @@ asdf-install:
 	asdf plugin update --all
 	asdf install || true
 	asdf reshim
+	pip install -r requirements.txt
+	asdf reshim
 
 # To copy data from the live environment, please follow these steps:
 # 1. Log into the OCP API using the API token provided by the OCP console.
diff --git a/helm/tools/dags/acs_dev.py b/helm/tools/dags/acs_dev.py
@@ -19,5 +19,5 @@
         task_id="fetch-load-acs-projects-dev",
         python_callable=fetch_load_acs_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
-        on_failure_callback=lambda context: send_alert(context, "acs_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/acs_prod.py b/helm/tools/dags/acs_prod.py
@@ -19,5 +19,5 @@
         task_id="fetch-load-acs-projects-prod",
         python_callable=fetch_load_acs_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
-        on_failure_callback=lambda context: send_alert(context, "acs_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/acs_test.py b/helm/tools/dags/acs_test.py
@@ -19,5 +19,5 @@
         task_id="fetch-load-acs-projects-test",
         python_callable=fetch_load_acs_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
-        on_failure_callback=lambda context: send_alert(context, "acs_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/airflow_failure_test.py b/helm/tools/dags/airflow_failure_test.py
@@ -32,5 +32,5 @@ def test_dag_task():
     t1 = PythonOperator(
         task_id="test_dag_task",
         python_callable=test_dag_task,
-        on_failure_callback=lambda context: send_alert(context, "airflow_failure_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/mailchimp_subscription_dev.py b/helm/tools/dags/mailchimp_subscription_dev.py
@@ -30,5 +30,5 @@
             "tag_id": MAILCHIMP_REGISTRY_PRIVATE_TAG_ID,
             "mongo_conn_id": MONGO_CONN_ID,
         },
-        on_failure_callback=lambda context: send_alert(context, "mailchimp_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/mailchimp_subscription_prod.py b/helm/tools/dags/mailchimp_subscription_prod.py
@@ -30,5 +30,5 @@
             "tag_id": MAILCHIMP_REGISTRY_PRIVATE_TAG_ID,
             "mongo_conn_id": MONGO_CONN_ID,
         },
-        on_failure_callback=lambda context: send_alert(context, "mailchimp_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/mailchimp_subscription_test.py b/helm/tools/dags/mailchimp_subscription_test.py
@@ -30,5 +30,5 @@
             "tag_id": MAILCHIMP_REGISTRY_PRIVATE_TAG_ID,
             "mongo_conn_id": MONGO_CONN_ID,
         },
-        on_failure_callback=lambda context: send_alert(context, "mailchimp_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/provisioner_prod.py b/helm/tools/dags/provisioner_prod.py
@@ -26,5 +26,5 @@
             "kc_client_id": REGISTRY_PROVISION_SA_ID,
             "kc_client_secret": REGISTRY_PROVISION_SA_SECRET,
         },
-        on_failure_callback=lambda context: send_alert(context, "provisioner_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/provisioner_test.py b/helm/tools/dags/provisioner_test.py
@@ -26,5 +26,5 @@
             "kc_client_id": REGISTRY_PROVISION_SA_ID,
             "kc_client_secret": REGISTRY_PROVISION_SA_SECRET,
         },
-        on_failure_callback=lambda context: send_alert(context, "provisioner_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/sonarscan_dev.py b/helm/tools/dags/sonarscan_dev.py
@@ -27,7 +27,7 @@
         task_id="fetch-sonarscan-projects-dev",
         python_callable=fetch_sonarscan_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID, "concurrency": CONCURRENCY, "gh_token": os.environ["GH_TOKEN"]},
-        on_failure_callback=lambda context: send_alert(context, "sonarscan_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
 
     shared_volume = V1Volume(
@@ -80,7 +80,7 @@
         python_callable=load_sonarscan_results,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
         trigger_rule=TriggerRule.ALL_DONE,
-        on_failure_callback=lambda context: send_alert(context, "sonarscan_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
         dag=dag,
     )
 
diff --git a/helm/tools/dags/sonarscan_prod.py b/helm/tools/dags/sonarscan_prod.py
@@ -27,7 +27,7 @@
         task_id="fetch-sonarscan-projects-prod",
         python_callable=fetch_sonarscan_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID, "concurrency": CONCURRENCY, "gh_token": os.environ["GH_TOKEN"]},
-        on_failure_callback=lambda context: send_alert(context, "sonarscan_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
 
     shared_volume = V1Volume(
@@ -80,7 +80,7 @@
         python_callable=load_sonarscan_results,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
         trigger_rule=TriggerRule.ALL_DONE,
-        on_failure_callback=lambda context: send_alert(context, "sonarscan_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
         dag=dag,
     )
 
diff --git a/helm/tools/dags/sonarscan_test.py b/helm/tools/dags/sonarscan_test.py
@@ -27,7 +27,7 @@
         task_id="fetch-sonarscan-projects-test",
         python_callable=fetch_sonarscan_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID, "concurrency": CONCURRENCY, "gh_token": os.environ["GH_TOKEN"]},
-        on_failure_callback=lambda context: send_alert(context, "sonarscan_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
 
     shared_volume = V1Volume(
@@ -80,7 +80,7 @@
         python_callable=load_sonarscan_results,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
         trigger_rule=TriggerRule.ALL_DONE,
-        on_failure_callback=lambda context: send_alert(context, "sonarscan_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
         dag=dag,
     )
 
diff --git a/helm/tools/dags/sync_db_with_azure_ad_dev.py b/helm/tools/dags/sync_db_with_azure_ad_dev.py
@@ -25,5 +25,5 @@
             "ms_graph_api_client_id": MS_GRAPH_API_CLIENT_ID,
             "ms_graph_api_client_secret": MS_GRAPH_API_CLIENT_SECRET,
         },
-        on_failure_callback=lambda context: send_alert(context, "sync_user_dbs_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/sync_db_with_azure_ad_prod.py b/helm/tools/dags/sync_db_with_azure_ad_prod.py
@@ -26,5 +26,5 @@
             "ms_graph_api_client_id": MS_GRAPH_API_CLIENT_ID,
             "ms_graph_api_client_secret": MS_GRAPH_API_CLIENT_SECRET,
         },
-        on_failure_callback=lambda context: send_alert(context, "sync_user_dbs_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/sync_db_with_azure_ad_test.py b/helm/tools/dags/sync_db_with_azure_ad_test.py
@@ -25,5 +25,5 @@
             "ms_graph_api_client_id": MS_GRAPH_API_CLIENT_ID,
             "ms_graph_api_client_secret": MS_GRAPH_API_CLIENT_SECRET,
         },
-        on_failure_callback=lambda context: send_alert(context, "sync_user_dbs_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/temporary_products_deletion_dev.py b/helm/tools/dags/temporary_products_deletion_dev.py
@@ -32,5 +32,5 @@
             "mongo_conn_id": MONGO_CONN_ID,
             "product_deletion_url_template": REGISTRY_DELETE_URL_TEMPLATE,
         },
-        on_failure_callback=lambda context: send_alert(context, "temporary_products_deletion_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/temporary_products_deletion_prod.py b/helm/tools/dags/temporary_products_deletion_prod.py
@@ -32,5 +32,5 @@
             "mongo_conn_id": MONGO_CONN_ID,
             "product_deletion_url_template": REGISTRY_DELETE_URL_TEMPLATE,
         },
-        on_failure_callback=lambda context: send_alert(context, "temporary_products_deletion_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/temporary_products_deletion_test.py b/helm/tools/dags/temporary_products_deletion_test.py
@@ -32,5 +32,5 @@
             "mongo_conn_id": MONGO_CONN_ID,
             "product_deletion_url_template": REGISTRY_DELETE_URL_TEMPLATE,
         },
-        on_failure_callback=lambda context: send_alert(context, "temporary_products_deletion_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/temporary_products_notification_dev.py b/helm/tools/dags/temporary_products_notification_dev.py
@@ -34,5 +34,5 @@
             "ches_api_url": CHES_API_URL,
             "app_url": APP_URL,
         },
-        on_failure_callback=lambda context: send_alert(context, "temporary_products_notification_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/temporary_products_notification_prod.py b/helm/tools/dags/temporary_products_notification_prod.py
@@ -34,5 +34,5 @@
             "ches_api_url": CHES_API_URL,
             "app_url": APP_URL,
         },
-        on_failure_callback=lambda context: send_alert(context, "temporary_products_notification_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/temporary_products_notification_test.py b/helm/tools/dags/temporary_products_notification_test.py
@@ -34,5 +34,5 @@
             "ches_api_url": CHES_API_URL,
             "app_url": APP_URL,
         },
-        on_failure_callback=lambda context: send_alert(context, "temporary_products_notification_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
diff --git a/helm/tools/dags/zapscan_dev.py b/helm/tools/dags/zapscan_dev.py
@@ -26,7 +26,7 @@
         task_id="fetch-zapscan-projects-dev",
         python_callable=fetch_zap_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID, "concurrency": CONCURRENCY},
-        on_failure_callback=lambda context: send_alert(context, "zapscan_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
 
     shared_volume = V1Volume(
@@ -72,7 +72,7 @@
         python_callable=load_zap_results,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
         trigger_rule=TriggerRule.ALL_DONE,
-        on_failure_callback=lambda context: send_alert(context, "zapscan_dev"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
         dag=dag,
     )
 
diff --git a/helm/tools/dags/zapscan_prod.py b/helm/tools/dags/zapscan_prod.py
@@ -26,7 +26,7 @@
         task_id="fetch-zapscan-projects-prod",
         python_callable=fetch_zap_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID, "concurrency": CONCURRENCY},
-        on_failure_callback=lambda context: send_alert(context, "zapscan_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
 
     shared_volume = V1Volume(
@@ -74,7 +74,7 @@
         python_callable=load_zap_results,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
         trigger_rule=TriggerRule.ALL_DONE,
-        on_failure_callback=lambda context: send_alert(context, "zapscan_prod"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
         dag=dag,
     )
 
diff --git a/helm/tools/dags/zapscan_test.py b/helm/tools/dags/zapscan_test.py
@@ -26,7 +26,7 @@
         task_id="fetch-zapscan-projects-test",
         python_callable=fetch_zap_projects,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID, "concurrency": CONCURRENCY},
-        on_failure_callback=lambda context: send_alert(context, "zapscan_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
     )
 
     shared_volume = V1Volume(
@@ -74,7 +74,7 @@
         python_callable=load_zap_results,
         op_kwargs={"mongo_conn_id": MONGO_CONN_ID},
         trigger_rule=TriggerRule.ALL_DONE,
-        on_failure_callback=lambda context: send_alert(context, "zapscan_test"),
+        on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),
         dag=dag,
     )
 

Original file line number	Diff line number	Diff line change
`@@ -19,5 +19,5 @@`
`19`	`19`	`task_id="fetch-load-acs-projects-dev",`
`20`	`20`	`python_callable=fetch_load_acs_projects,`
`21`	`21`	`op_kwargs={"mongo_conn_id": MONGO_CONN_ID},`
`22`		`- on_failure_callback=lambda context: send_alert(context, "acs_dev"),`
	`22`	`+ on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),`
`23`	`23`	`)`
Original file line number	Diff line number	Diff line change
`@@ -32,5 +32,5 @@ def test_dag_task():`
`32`	`32`	`t1 = PythonOperator(`
`33`	`33`	`task_id="test_dag_task",`
`34`	`34`	`python_callable=test_dag_task,`
`35`		`- on_failure_callback=lambda context: send_alert(context, "airflow_failure_test"),`
	`35`	`+ on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),`
`36`	`36`	`)`
Original file line number	Diff line number	Diff line change
`@@ -30,5 +30,5 @@`
`30`	`30`	`"tag_id": MAILCHIMP_REGISTRY_PRIVATE_TAG_ID,`
`31`	`31`	`"mongo_conn_id": MONGO_CONN_ID,`
`32`	`32`	`},`
`33`		`- on_failure_callback=lambda context: send_alert(context, "mailchimp_dev"),`
	`33`	`+ on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),`
`34`	`34`	`)`
Original file line number	Diff line number	Diff line change
`@@ -26,5 +26,5 @@`
`26`	`26`	`"kc_client_id": REGISTRY_PROVISION_SA_ID,`
`27`	`27`	`"kc_client_secret": REGISTRY_PROVISION_SA_SECRET,`
`28`	`28`	`},`
`29`		`- on_failure_callback=lambda context: send_alert(context, "provisioner_prod"),`
	`29`	`+ on_failure_callback=lambda context: send_alert(context, context["dag"].dag_id),`
`30`	`30`	`)`