Make failures more visible (#179)

- *Category*:  feature
- *JIRA issue*: [MIC-4265](https://jira.ihme.washington.edu/browse/MIC-4265)

Changes and notes
- Adds a warning at psimulate epilogue if there were failed jobs
- Updates final job completion log message to include status (finished/total)

Author: mattkappel

CHANGELOG.rst

@@ -1,3 +1,7 @@
+**1.3.11 - 09/07/23**
+
+ - Made job failures more prominent in end of jobs logging
+
 **1.3.10 - 07/12/23**
 
  - Allow for specifying random seeds and draws in branches file

src/vivarium_cluster_tools/psimulate/runner.py

@@ -6,7 +6,7 @@
 The main process loop for `psimulate` runs.
 
 """
-import atexit
+from collections import defaultdict
 from pathlib import Path
 from time import sleep, time
 
@@ -34,10 +34,11 @@ def process_job_results(
     existing_outputs: pd.DataFrame,
     output_directory: Path,
     no_batch: bool,
-) -> None:
+) -> defaultdict:
     written_results = existing_outputs
     unwritten_results = []
     batch_size = 0 if no_batch else 200
+    status = defaultdict(int)
 
     logger.info("Entering main processing loop.")
     start_time = time()
@@ -54,7 +55,7 @@ def process_job_results(
                     batch_size,
                 )
 
-            registry_manager.update_and_report()
+            status = registry_manager.update_and_report()
             logger.info(f"Unwritten results: {len(unwritten_results)}")
             logger.info(f"Elapsed time: {(time() - start_time)/60:.1f} minutes.")
     finally:
@@ -68,6 +69,7 @@ def process_job_results(
             )
             logger.info(f"Unwritten results: {len(unwritten_results)}")
             logger.info(f"Elapsed time: {(time() - start_time) / 60:.1f} minutes.")
+        return status
 
 
 def load_existing_outputs(result_path: Path, restart: bool) -> pd.DataFrame:
@@ -238,7 +240,7 @@ def main(
     # Enter the main monitoring and processing loop, which will check on
     # all the queues periodically, report status updates, and gather
     # and write results when they are available.
-    process_job_results(
+    status = process_job_results(
         registry_manager=registry_manager,
         existing_outputs=existing_outputs,
         output_directory=output_paths.root,
@@ -248,4 +250,14 @@ def main(
     # Spit out a performance report for the workers.
     try_run_vipin(output_paths.worker_logging_root)
 
-    logger.info(f"Jobs completed. Results written to: {str(output_paths.root)}")
+    # Emit warning if any jobs failed
+    if status["failed"] > 0:
+        logger.warning(
+            f"*** NOTE: There {'was' if status['failed'] == 1 else 'were'} "
+            f"{status['failed']} failed job{'' if status['failed'] == 1 else 's'}. ***"
+        )
+
+    logger.info(
+        f"{status['finished']} of {status['total']} jobs completed successfully. "
+        f"Results written to: {str(output_paths.root)}"
+    )