OPAL: re-enable async progress thread

Minh Quan Ho · Minh Quan Ho · commit 70e3bf2bfebf · 2025-07-09T15:51:10.000+02:00
- The SW-based async progress thread has been planned long time ago in 683efcb, but has never been enabled/implemented since. - This commit enables the spawn of an async progress thread to execute _opal_progress() routine when enabled at both compile time and runtime (--enable-progress-threads (default=enabled) and MCA variables opal_async_progress or mpi_async_progress). - Fix minor typo in opal_progress.h doxygen comment Signed-off-by: Minh Quan Ho <minh-quan.ho@sipearl.com>
diff --git a/config/opal_configure_options.m4 b/config/opal_configure_options.m4
@@ -544,9 +544,21 @@ fi
 AC_DEFINE_UNQUOTED([OPAL_ENABLE_GETPWUID], [$opal_want_getpwuid],
                    [Disable getpwuid support (default: enabled)])
 
-dnl We no longer support the old OPAL_ENABLE_PROGRESS_THREADS.  At
-dnl some point, this should die.
-AC_DEFINE([OPAL_ENABLE_PROGRESS_THREADS],
-          [0],
-          [Whether we want BTL progress threads enabled])
+#
+# Disable progress threads
+#
+AC_MSG_CHECKING([if want asynchronous progress threads])
+AC_ARG_ENABLE([progress_threads],
+    [AS_HELP_STRING([--disable-progress-threads],
+                   [Disable asynchronous progress threads. Note that when enabled, for performance-related reasons, the progress thread is still not spawned by default. User must enable MCA variables 'opal_async_progress' or 'mpi_async_progress' to have the progress thread spawned at runtime. (default: enabled)])])
+if test "$enable_progress_threads" = "no"; then
+    AC_MSG_RESULT([no])
+    opal_want_progress_threads=0
+else
+    AC_MSG_RESULT([yes])
+    opal_want_progress_threads=1
+fi
+AC_DEFINE_UNQUOTED([OPAL_ENABLE_PROGRESS_THREADS], [$opal_want_progress_threads],
+    [Disable BTL asynchronous progress threads (default: enabled)])
+
 ])dnl
diff --git a/ompi/instance/instance.c b/ompi/instance/instance.c
@@ -512,7 +512,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
        ddt_init, but before mca_coll_base_open, since some collective
        modules (e.g., the hierarchical coll component) may need ops in
        their query function. */
-    if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+    if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {
         return ompi_instance_print_error ("ompi_op_base_find_available() failed", ret);
     }
 
@@ -532,7 +532,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("mca_smsc_base_select() failed", ret);
     }
 
-    if (OMPI_SUCCESS != (ret = mca_pml_base_select (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+    if (OMPI_SUCCESS != (ret = mca_pml_base_select (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {
         return ompi_instance_print_error ("mca_pml_base_select() failed", ret);
     }
 
@@ -617,7 +617,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
         return ompi_instance_print_error ("mca_pml_base_bsend_init() failed", ret);
     }
 
-    if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {
+    if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {
         return ompi_instance_print_error ("mca_coll_base_find_available() failed", ret);
     }
 
diff --git a/ompi/mpi/c/request_get_status.c.in b/ompi/mpi/c/request_get_status.c.in
@@ -38,9 +38,7 @@
 PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
                                          STATUS_OUT status)
 {
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
     int do_it_once = 0;
-#endif
 
     MEMCHECKER(
         memchecker_request(&request);
@@ -56,9 +54,7 @@ PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
         }
     }
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
  recheck_request_status:
-#endif
     opal_atomic_mb();
     if( (request == MPI_REQUEST_NULL) || (request->req_state == OMPI_REQUEST_INACTIVE) ) {
         *flag = true;
@@ -80,16 +76,16 @@ PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,
         }
         return MPI_SUCCESS;
     }
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    if( 0 == do_it_once ) {
+
+    if( 0 == do_it_once && !opal_async_progress_thread_spawned ) {
         /* If we run the opal_progress then check the status of the
            request before leaving. We will call the opal_progress only
            once per call. */
         opal_progress();
         do_it_once++;
         goto recheck_request_status;
     }
-#endif
+
     *flag = false;
     return MPI_SUCCESS;
 }
diff --git a/ompi/request/req_test.c b/ompi/request/req_test.c
@@ -32,11 +32,9 @@ int ompi_request_default_test(ompi_request_t ** rptr,
 {
     ompi_request_t *request = *rptr;
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
     int do_it_once = 0;
 
 recheck_request_status:
-#endif
     if( request->req_state == OMPI_REQUEST_INACTIVE ) {
         *completed = true;
         if (MPI_STATUS_IGNORE != status) {
@@ -81,8 +79,8 @@ int ompi_request_default_test(ompi_request_t ** rptr,
         return MPI_ERR_PROC_FAILED_PENDING;
     }
 #endif
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    if( 0 == do_it_once ) {
+
+    if( 0 == do_it_once && !opal_async_progress_thread_spawned ) {
         /**
          * If we run the opal_progress then check the status of the request before
          * leaving. We will call the opal_progress only once per call.
@@ -92,7 +90,7 @@ int ompi_request_default_test(ompi_request_t ** rptr,
             goto recheck_request_status;
         }
     }
-#endif
+
     *completed = false;
     return OMPI_SUCCESS;
 }
@@ -163,9 +161,9 @@ int ompi_request_default_test_any(
     *index = MPI_UNDEFINED;
     if(num_requests_null_inactive != count) {
         *completed = false;
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        opal_progress();
-#endif
+        if (!opal_async_progress_thread_spawned) {
+            opal_progress();
+        }
     } else {
         *completed = true;
         if (MPI_STATUS_IGNORE != status) {
@@ -208,16 +206,16 @@ int ompi_request_default_test_all(
             return MPI_ERR_PROC_FAILED_PENDING;
         }
 #endif /* OPAL_ENABLE_FT_MPI */
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        if (0 == do_it_once) {
+
+        if (0 == do_it_once && !opal_async_progress_thread_spawned) {
             ++do_it_once;
             if (0 != opal_progress()) {
                 /* continue walking the list, retest the current request */
                 --i;
                 continue;
             }
         }
-#endif /* OPAL_ENABLE_PROGRESS_THREADS */
+
         /* short-circuit */
         break;
     }
@@ -353,9 +351,9 @@ int ompi_request_default_test_some(
     *outcount = num_requests_done;
 
     if (num_requests_done == 0) {
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-        opal_progress();
-#endif
+        if (!opal_async_progress_thread_spawned) {
+            opal_progress();
+        }
         return OMPI_SUCCESS;
     }
 
diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c
@@ -193,9 +193,10 @@ int ompi_mpi_finalize(void)
     opal_atomic_swap_32(&ompi_mpi_state,
                         OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
-    opal_progress_set_event_flag(OPAL_EVLOOP_ONCE | OPAL_EVLOOP_NONBLOCK);
-#endif
+    /* shutdown async progress thread before tearing down further services */
+    if (opal_async_progress_thread_spawned) {
+        opal_progress_shutdown_async_progress_thread();
+    }
 
     /* NOTE: MPI-2.1 requires that MPI_FINALIZE is "collective" across
        *all* connected processes.  This only means that all processes
diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c
@@ -559,16 +559,16 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
        time if so, then start the clock again */
     OMPI_TIMING_NEXT("barrier");
 
-#if OPAL_ENABLE_PROGRESS_THREADS == 0
     /* Start setting up the event engine for MPI operations.  Don't
        block in the event library, so that communications don't take
        forever between procs in the dynamic code.  This will increase
        CPU utilization for the remainder of MPI_INIT when we are
        blocking on RTE-level events, but may greatly reduce non-TCP
        latency. */
-    int old_event_flags = opal_progress_set_event_flag(0);
-    opal_progress_set_event_flag(old_event_flags | OPAL_EVLOOP_NONBLOCK);
-#endif
+    if (!opal_async_progress_thread_spawned) {
+        int old_event_flags = opal_progress_set_event_flag(0);
+        opal_progress_set_event_flag(old_event_flags | OPAL_EVLOOP_NONBLOCK);
+    }
 
     /* wire up the mpi interface, if requested.  Do this after the
        non-block switch for non-TCP performance.  Do before the
diff --git a/ompi/runtime/ompi_mpi_params.c b/ompi/runtime/ompi_mpi_params.c
@@ -351,6 +351,13 @@ int ompi_mpi_register_params(void)
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &ompi_async_mpi_finalize);
 
+#if OPAL_ENABLE_PROGRESS_THREADS == 1
+    value = mca_base_var_find ("opal", "opal", NULL, "async_progress");
+    if (0 <= value) {
+        (void) mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "async_progress", 0);
+    }
+#endif
+
     value = mca_base_var_find ("opal", "opal", NULL, "abort_delay");
     if (0 <= value) {
         (void) mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "abort_delay",
diff --git a/opal/mca/btl/smcuda/btl_smcuda.c b/opal/mca/btl/smcuda/btl_smcuda.c
@@ -487,7 +487,7 @@ static struct mca_btl_base_endpoint_t *create_sm_endpoint(int local_proc, struct
     OBJ_CONSTRUCT(&ep->endpoint_lock, opal_mutex_t);
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
     sprintf(path, "%s" OPAL_PATH_SEP "sm_fifo.%lu", opal_process_info.job_session_dir,
-            (unsigned long) proc->proc_name);
+            (unsigned long) proc->proc_name.vpid);
     ep->fifo_fd = open(path, O_WRONLY);
     if (ep->fifo_fd < 0) {
         opal_output(0, "mca_btl_smcuda_add_procs: open(%s) failed with errno=%d\n", path, errno);
diff --git a/opal/mca/btl/smcuda/btl_smcuda_component.c b/opal/mca/btl/smcuda/btl_smcuda_component.c
@@ -860,7 +860,7 @@ mca_btl_smcuda_component_init(int *num_btls, bool enable_progress_threads, bool
 #if OPAL_ENABLE_PROGRESS_THREADS == 1
     /* create a named pipe to receive events  */
     sprintf(mca_btl_smcuda_component.sm_fifo_path, "%s" OPAL_PATH_SEP "sm_fifo.%lu",
-            opal_process_info.job_session_dir, (unsigned long) OPAL_PROC_MY_NAME->vpid);
+            opal_process_info.job_session_dir, (unsigned long) OPAL_PROC_MY_NAME.vpid);
     if (mkfifo(mca_btl_smcuda_component.sm_fifo_path, 0660) < 0) {
         opal_output(0, "mca_btl_smcuda_component_init: mkfifo failed with errno=%d\n", errno);
         return NULL;
diff --git a/opal/runtime/opal_params_core.c b/opal/runtime/opal_params_core.c
@@ -90,6 +90,10 @@ int opal_abort_delay = 0;
 
 int opal_max_thread_in_progress = 1;
 
+#if OPAL_ENABLE_PROGRESS_THREADS == 1
+bool opal_async_progress = false;
+#endif
+
 static bool opal_register_util_done = false;
 
 static char *opal_var_dump_color_string = NULL;
@@ -416,6 +420,14 @@ int opal_register_util_params(void)
                                  MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8,
                                  MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress);
 
+#if OPAL_ENABLE_PROGRESS_THREADS == 1
+    /* Spawn a dedicated software progress-thread to execute opal_progress() */
+    (void) mca_base_var_register("opal", "opal", NULL, "async_progress",
+                                 "Spawn a dedicated software progress-thread. Default: false",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY, &opal_async_progress);
+#endif
+
     /* Use sync_memops functionality with accelerator codes or deploy
        alternative path using IPC events to ensure consistency */
     opal_accelerator_use_sync_memops = true;
diff --git a/opal/runtime/opal_params_core.h b/opal/runtime/opal_params_core.h
@@ -60,6 +60,16 @@ OPAL_DECLSPEC extern bool opal_built_with_cuda_support;
 OPAL_DECLSPEC extern bool opal_built_with_rocm_support;
 OPAL_DECLSPEC extern bool opal_built_with_ze_support;
 
+#    if OPAL_ENABLE_PROGRESS_THREADS == 1
+OPAL_DECLSPEC extern bool opal_async_progress_thread_spawned;
+#    else
+/* When disabled at configure, using 'static const' will allow compilers to
+ * not only evaluate the boolean at compile time, but also to remove its
+ * storage within any object file which is including this header.
+ * See https://godbolt.org/z/hxc8EjPa4  */
+OPAL_DECLSPEC static const bool opal_async_progress_thread_spawned = false;
+#    endif
+
 /**
  *  * Whether we want to enable CUDA GPU buffer send and receive support.
  *   */
diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c
diff --git a/opal/runtime/opal_progress.h b/opal/runtime/opal_progress.h

Original file line number	Diff line number	Diff line change
`@@ -512,7 +512,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)`
`512`	`512`	`ddt_init, but before mca_coll_base_open, since some collective`
`513`	`513`	`modules (e.g., the hierarchical coll component) may need ops in`
`514`	`514`	`their query function. */`
`515`		`- if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {`
	`515`	`+ if (OMPI_SUCCESS != (ret = ompi_op_base_find_available (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {`
`516`	`516`	`return ompi_instance_print_error ("ompi_op_base_find_available() failed", ret);`
`517`	`517`	`}`
`518`	`518`
`@@ -532,7 +532,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)`
`532`	`532`	`return ompi_instance_print_error ("mca_smsc_base_select() failed", ret);`
`533`	`533`	`}`
`534`	`534`
`535`		`- if (OMPI_SUCCESS != (ret = mca_pml_base_select (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {`
	`535`	`+ if (OMPI_SUCCESS != (ret = mca_pml_base_select (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {`
`536`	`536`	`return ompi_instance_print_error ("mca_pml_base_select() failed", ret);`
`537`	`537`	`}`
`538`	`538`
`@@ -617,7 +617,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)`
`617`	`617`	`return ompi_instance_print_error ("mca_pml_base_bsend_init() failed", ret);`
`618`	`618`	`}`
`619`	`619`
`620`		`- if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (OPAL_ENABLE_PROGRESS_THREADS, ompi_mpi_thread_multiple))) {`
	`620`	`+ if (OMPI_SUCCESS != (ret = mca_coll_base_find_available (opal_async_progress_thread_spawned, ompi_mpi_thread_multiple))) {`
`621`	`621`	`return ompi_instance_print_error ("mca_coll_base_find_available() failed", ret);`
`622`	`622`	`}`
`623`	`623`
Original file line number	Diff line number	Diff line change
`@@ -38,9 +38,7 @@`
`38`	`38`	`PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,`
`39`	`39`	`STATUS_OUT status)`
`40`	`40`	`{`
`41`		`-#if OPAL_ENABLE_PROGRESS_THREADS == 0`
`42`	`41`	`int do_it_once = 0;`
`43`		`-#endif`
`44`	`42`
`45`	`43`	`MEMCHECKER(`
`46`	`44`	`memchecker_request(&request);`
`@@ -56,9 +54,7 @@ PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,`
`56`	`54`	`}`
`57`	`55`	`}`
`58`	`56`
`59`		`-#if OPAL_ENABLE_PROGRESS_THREADS == 0`
`60`	`57`	`recheck_request_status:`
`61`		`-#endif`
`62`	`58`	`opal_atomic_mb();`
`63`	`59`	`if( (request == MPI_REQUEST_NULL) \|\| (request->req_state == OMPI_REQUEST_INACTIVE) ) {`
`64`	`60`	`*flag = true;`
`@@ -80,16 +76,16 @@ PROTOTYPE ERROR_CLASS request_get_status(REQUEST request, INT_OUT flag,`
`80`	`76`	`}`
`81`	`77`	`return MPI_SUCCESS;`
`82`	`78`	`}`
`83`		`-#if OPAL_ENABLE_PROGRESS_THREADS == 0`
`84`		`- if( 0 == do_it_once ) {`
	`79`	`+`
	`80`	`+ if( 0 == do_it_once && !opal_async_progress_thread_spawned ) {`
`85`	`81`	`/* If we run the opal_progress then check the status of the`
`86`	`82`	`request before leaving. We will call the opal_progress only`
`87`	`83`	`once per call. */`
`88`	`84`	`opal_progress();`
`89`	`85`	`do_it_once++;`
`90`	`86`	`goto recheck_request_status;`
`91`	`87`	`}`
`92`		`-#endif`
	`88`	`+`
`93`	`89`	`*flag = false;`
`94`	`90`	`return MPI_SUCCESS;`
`95`	`91`	`}`