Skip to content

Commit 372cf36

Browse files
markbrown314parkerha1
markbrown314
and
parkerha1
committed
Multiple Initialization and Finalization Support
* Support for multiple calls for shmem_init, shmem_init_thread, and shmem_finalize * Added API shmemx_query_initialzed() * Based off of a prototype from Parker Harris Issue #1171 Signed-off-by: Mark F. Brown <mark.f.brown@intel.com> Co-authored-by: Parker Harris <parker.harris@intel.com>
1 parent 31a58aa commit 372cf36

File tree

7 files changed

+91
-49
lines changed

7 files changed

+91
-49
lines changed

‎configure.ac‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,11 @@ AC_ARG_ENABLE([ofi-hmem],
340340
AS_IF([test "$enable_ofi_hmem" = "yes"],
341341
[AC_DEFINE([USE_FI_HMEM], [1], [If defined, the OFI transport will enable FI_HMEM.])])
342342

343+
AC_ARG_ENABLE([multi-init],
344+
[AS_HELP_STRING([--disable-multi-init],
345+
[Disable multiple initialization and finalization feature. (default: enabled)])]])
346+
347+
343348
PKG_INSTALLDIR()
344349

345350
dnl check for programs
@@ -1002,7 +1007,13 @@ AC_CONFIG_FILES([Makefile
10021007
modules/tests-sos/test/performance/tests/Makefile
10031008
modules/tests-sos/test/apps/Makefile])
10041009

1005-
AC_OUTPUT
1010+
#AS_IF([test "$enable_pmi_mpi" = "yes" -a "$enable_multi_init" = "yes"],
1011+
# [AC_MSG_ERROR([Multiple initilization and finalization is not supported with PMI MPI])])
1012+
1013+
AM_CONDITIONAL([ENABLE_MULTI_INIT], [test "$enable_multi_init" = "yes" -a "$enable_pmi_mpi" != "yes"])
1014+
1015+
AS_IF([test "$enable_pmi_mpi" != "yes" -a "$enable_multi_init" = "yes"],
1016+
[AC_DEFINE([USE_MULTI_INIT], [1], [If defined, multiple init will be supported.])])
10061017

10071018
AS_IF([test "$enable_pmi_mpi" != "yes" -a "$enable_pmi_simple" != "yes" -a "$opal_external_pmix_version_found" != 1 -a "$ompi_check_pmi_happy" != "yes" -a -z "$pmi_type"],
10081019
[AC_MSG_ERROR([No PMI client interface was configured, consider --enable-pmi-simple or --with-pmi])])
@@ -1025,6 +1036,8 @@ AS_IF([test "$enable_fortran" = "yes" -a "$FC" = ""],
10251036
AS_IF([test "$enable_fortran" = "yes"],
10261037
[AC_MSG_WARN([The Fortran API was deprecated in OpenSHMEM v1.4 and removed in OpenSHMEM v1.5])])
10271038

1039+
AC_OUTPUT
1040+
10281041
FORT="$FC"
10291042
if test "$FORT" = "" ; then
10301043
FORT="none"

‎mpp/shmemx_c_func.h4‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,6 @@ SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_create(void *base, size_t size
105105
SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_preinit(void);
106106
SHMEM_FUNCTION_ATTRIBUTES int SHPRE()shmemx_heap_preinit_thread(int requested, int *provided);
107107
SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_heap_postinit(void);
108+
109+
/* Support querying intialization status */
110+
SHMEM_FUNCTION_ATTRIBUTES void SHPRE()shmemx_query_initialized(int *initialized);

‎src/init.c‎

Lines changed: 55 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <stdio.h>
2929
#include <limits.h>
3030
#include <string.h>
31+
#include <pthread.h>
3132

3233
#define SHMEM_INTERNAL_INCLUDE
3334
#include "shmem.h"
@@ -37,6 +38,7 @@
3738
#include "runtime.h"
3839
#include "build_info.h"
3940
#include "shmem_team.h"
41+
#include "shmem_atomic.h"
4042

4143
#if defined(ENABLE_REMOTE_VIRTUAL_ADDRESSING) && defined(__linux__)
4244
#include <sys/personality.h>
@@ -89,10 +91,10 @@ int shmem_external_heap_device = -1;
8991

9092
int shmem_internal_my_pe = -1;
9193
int shmem_internal_num_pes = -1;
92-
int shmem_internal_initialized = 0;
93-
int shmem_internal_finalized = 0;
94+
shmem_internal_cntr_t shmem_internal_init_cntr = 0;
9495
int shmem_internal_initialized_with_start_pes = 0;
9596
int shmem_internal_global_exit_called = 0;
97+
int shmem_internal_inited_once = 0;
9698

9799
int shmem_internal_thread_level;
98100

@@ -134,44 +136,58 @@ shmem_internal_randr_fini(void)
134136
return;
135137
}
136138

137-
138139
static void
139-
shmem_internal_shutdown(void)
140+
shmem_release_resources(void)
140141
{
141-
if (!shmem_internal_initialized ||
142-
shmem_internal_finalized) {
143-
return;
142+
if (shmem_internal_params.DEBUG) {
143+
DEBUG_MSG("releasing resources\n");
144144
}
145145

146-
shmem_internal_barrier_all();
147-
148-
shmem_internal_finalized = 1;
149-
150146
shmem_internal_team_fini();
151-
152147
shmem_transport_fini();
153-
154148
shmem_shr_transport_fini();
155-
156149
SHMEM_MUTEX_DESTROY(shmem_internal_mutex_alloc);
157-
158150
shmem_internal_randr_fini();
159-
160151
shmem_internal_symmetric_fini();
161152
shmem_runtime_fini();
162153
}
163154

164155

156+
static void
157+
shmem_internal_shutdown(void)
158+
{
159+
if (!shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
160+
RAISE_WARN_STR("too many calls to shmem_finalize");
161+
return;
162+
}
163+
164+
shmem_internal_barrier_all();
165+
166+
#ifndef USE_MULTI_INIT
167+
shmem_release_resources();
168+
#else
169+
shmem_internal_cntr_dec(&shmem_internal_init_cntr);
170+
#endif
171+
172+
return;
173+
}
174+
175+
165176
static void
166177
shmem_internal_shutdown_atexit(void)
167178
{
168-
if ( shmem_internal_initialized && !shmem_internal_finalized &&
169-
!shmem_internal_initialized_with_start_pes && !shmem_internal_global_exit_called &&
170-
shmem_internal_my_pe == 0) {
171-
RAISE_WARN_STR("shutting down without a call to shmem_finalize");
179+
if (shmem_internal_cntr_read(&shmem_internal_init_cntr) &&
180+
!shmem_internal_initialized_with_start_pes &&
181+
!shmem_internal_global_exit_called &&
182+
shmem_internal_my_pe == 0) {
183+
184+
RAISE_WARN_STR("too few calls to shmem_finalize");
172185
}
173186

174-
shmem_internal_shutdown();
187+
#ifdef USE_MULTI_INIT
188+
shmem_release_resources();
189+
#endif
190+
175191
}
176192

177193

@@ -518,7 +534,6 @@ shmem_internal_heap_postinit(void)
518534
randr_initialized = 1;
519535

520536
atexit(shmem_internal_shutdown_atexit);
521-
shmem_internal_initialized = 1;
522537

523538
/* finish up */
524539
#ifndef USE_PMIX
@@ -554,11 +569,25 @@ shmem_internal_init(int tl_requested, int *tl_provided)
554569
{
555570
int ret;
556571

557-
ret = shmem_internal_heap_preinit(tl_requested, tl_provided);
558-
if (ret) goto cleanup;
572+
#ifndef USE_MULTI_INIT
573+
if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
574+
RAISE_ERROR_STR("attempt to reinitialize library");
575+
}
576+
#endif
559577

560-
ret = shmem_internal_heap_postinit();
561-
if (ret) goto cleanup;
578+
if (shmem_internal_cntr_fadd(&shmem_internal_init_cntr, 1) == 0
579+
&& !shmem_internal_inited_once) {
580+
581+
shmem_internal_inited_once = 1;
582+
583+
ret = shmem_internal_heap_preinit(tl_requested, tl_provided);
584+
if (ret) goto cleanup;
585+
586+
ret = shmem_internal_heap_postinit();
587+
if (ret) goto cleanup;
588+
} else {
589+
*tl_provided = shmem_internal_thread_level;
590+
}
562591

563592
return 0;
564593

‎src/init_c.c‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "shmem.h"
2121
#include "shmem_internal.h"
2222
#include "shmemx.h"
23+
#include "shmem_atomic.h"
2324

2425
#ifdef ENABLE_PROFILING
2526
#include "pshmem.h"
@@ -59,10 +60,13 @@
5960

6061
#endif /* ENABLE_PROFILING */
6162

63+
extern shmem_internal_cntr_t shmem_internal_init_cntr;
64+
65+
6266
void SHMEM_FUNCTION_ATTRIBUTES
6367
start_pes(int npes)
6468
{
65-
if (!shmem_internal_initialized) {
69+
if (!shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
6670
shmem_internal_start_pes(npes);
6771
}
6872
}
@@ -73,10 +77,6 @@ shmem_init(void)
7377
{
7478
int tl_provided, ret;
7579

76-
if (shmem_internal_initialized) {
77-
RAISE_ERROR_STR("attempt to reinitialize library");
78-
}
79-
8080
ret = shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided);
8181
if (ret) abort();
8282
}
@@ -87,7 +87,7 @@ shmemx_heap_preinit(void)
8787
{
8888
int tl_provided, ret;
8989

90-
if (shmem_internal_initialized) {
90+
if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
9191
RAISE_ERROR_STR("attempt to reinitialize library");
9292
}
9393

@@ -108,9 +108,6 @@ int SHMEM_FUNCTION_ATTRIBUTES
108108
shmem_init_thread(int tl_requested, int *tl_provided)
109109
{
110110
int ret;
111-
if (shmem_internal_initialized) {
112-
RAISE_ERROR_STR("attempt to reinitialize library");
113-
}
114111

115112
ret = shmem_internal_init(tl_requested, tl_provided);
116113
return ret;
@@ -121,7 +118,7 @@ int SHMEM_FUNCTION_ATTRIBUTES
121118
shmemx_heap_preinit_thread(int tl_requested, int *tl_provided)
122119
{
123120
int ret;
124-
if (shmem_internal_initialized) {
121+
if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
125122
RAISE_ERROR_STR("attempt to reinitialize library");
126123
}
127124

@@ -138,6 +135,11 @@ shmem_query_thread(int *provided)
138135
*provided = shmem_internal_thread_level;
139136
}
140137

138+
void SHMEM_FUNCTION_ATTRIBUTES
139+
shmemx_query_initialized(int *initialized)
140+
{
141+
*initialized = shmem_internal_cntr_read(&shmem_internal_init_cntr);
142+
}
141143

142144
void SHMEM_FUNCTION_ATTRIBUTES
143145
shmem_global_exit(int status)

‎src/init_f.c‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@ void SHMEM_FUNCTION_ATTRIBUTES FC_START_PES(fortran_integer_t *npes);
2626
void
2727
FC_START_PES(fortran_integer_t *npes)
2828
{
29-
if (shmem_internal_initialized) {
30-
RAISE_ERROR_STR("attempt to reinitialize library");
31-
}
32-
3329
shmem_internal_start_pes(*npes);
3430
}
3531

@@ -40,11 +36,6 @@ void
4036
FC_SHMEM_INIT(void)
4137
{
4238
int tl_provided;
43-
44-
if (shmem_internal_initialized) {
45-
RAISE_ERROR_STR("attempt to reinitialize library");
46-
}
47-
4839
shmem_internal_init(SHMEM_THREAD_SINGLE, &tl_provided);
4940
}
5041

‎src/shmem_internal.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
extern int shmem_internal_my_pe;
3434
extern int shmem_internal_num_pes;
3535

36-
extern int shmem_internal_initialized;
3736
extern int shmem_internal_finalized;
3837
extern int shmem_internal_thread_level;
3938

@@ -189,7 +188,9 @@ extern hwloc_topology_t shmem_internal_topology;
189188
#ifdef ENABLE_ERROR_CHECKING
190189
#define SHMEM_ERR_CHECK_INITIALIZED() \
191190
do { \
192-
if (!shmem_internal_initialized) { \
191+
int initialized = 0; \
192+
shmemx_query_initialized(&initialized); \
193+
if (!initialized) { \
193194
RETURN_ERROR_STR(PACKAGE_NAME " library not initialized\n"); \
194195
abort(); \
195196
} \

‎src/symmetric_heap_c.c‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "shmem_comm.h"
3535
#include "shmem_collectives.h"
3636
#include "shmemx.h"
37+
#include "shmem_atomic.h"
3738

3839
#ifdef ENABLE_PROFILING
3940
#include "pshmem.h"
@@ -73,6 +74,8 @@
7374

7475
#endif /* ENABLE_PROFILING */
7576

77+
extern shmem_internal_cntr_t shmem_internal_init_cntr;
78+
7679
static char *shmem_internal_heap_curr = NULL;
7780

7881
void* dlmalloc(size_t);
@@ -439,7 +442,7 @@ shmem_malloc_with_hints(size_t size, long hints)
439442
void SHMEM_FUNCTION_ATTRIBUTES
440443
shmemx_heap_create(void *base, size_t size, int device_type, int device_index) {
441444

442-
if (shmem_internal_initialized) {
445+
if (shmem_internal_cntr_read(&shmem_internal_init_cntr)) {
443446
RAISE_WARN_MSG("Ignoring pre-setup. Heap already initialized\n");
444447
return;
445448
}

0 commit comments

Comments
 (0)