Skip to content

Commit 1eae8d3

Browse files
wkliaowkliao
committed
new test program to test single large request from each process
1 parent 535e1e4 commit 1eae8d3

File tree

1 file changed

+355
-0
lines changed

1 file changed

+355
-0
lines changed

‎tests/large_dtype.c

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,355 @@
1+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2+
*
3+
* Copyright (C) 2024, Northwestern University
4+
*
5+
* This program tests large requests made by each MPI process.
6+
* Both buffer and fileview data types are noncontiguous.
7+
*
8+
* The local buffer datatype comprises NVARS arrays. Each array is 2D of size
9+
* len x len. A gap of size GAP can be added at the end of each dimension to
10+
* make the data type noncontiguous.
11+
*
12+
* The fileview of each process comprises NVARS subarrays. Each global array
13+
* is 2D of size (len * psize[0]) x (len * psize[1]). Each local array is of
14+
* size (len - GAP) x (len - GAP).
15+
*
16+
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
17+
18+
#include <stdio.h>
19+
#include <stdlib.h>
20+
#include <string.h> /* strcpy() */
21+
#include <unistd.h> /* getopt() */
22+
#include <assert.h>
23+
24+
#include <mpi.h>
25+
26+
#define CHECK_MPI_ERROR(fname) \
27+
if (err != MPI_SUCCESS) { \
28+
int errorStringLen; \
29+
char errorString[MPI_MAX_ERROR_STRING]; \
30+
MPI_Error_string(err, errorString, &errorStringLen); \
31+
printf("Error at line %d when calling %s: %s\n",__LINE__,fname,errorString); \
32+
}
33+
34+
#define CHECK_MPIO_ERROR(fname) { \
35+
if (err != MPI_SUCCESS) { \
36+
int errorStringLen; \
37+
char errorString[MPI_MAX_ERROR_STRING]; \
38+
MPI_Error_string(err, errorString, &errorStringLen); \
39+
printf("Error at line %d when calling %s: %s\n",__LINE__,fname,errorString); \
40+
} \
41+
else if (verbose) { \
42+
if (rank == 0) \
43+
printf("---- pass LINE %d of calling %s\n", __LINE__, fname); \
44+
fflush(stdout); \
45+
MPI_Barrier(MPI_COMM_WORLD); \
46+
} \
47+
}
48+
49+
#define LEN 2048
50+
#define GAP 1
51+
#define NVARS 1100
52+
53+
static void
54+
usage(char *argv0)
55+
{
56+
char *help =
57+
"Usage: %s [-hvwr | -n num | -l num | -g num ] -f file_name\n"
58+
" [-h] Print this help\n"
59+
" [-v] verbose mode\n"
60+
" [-w] performs write only (default: both write and read)\n"
61+
" [-r] performs read only (default: both write and read)\n"
62+
" [-n num] number of global variables (default: %d)\n"
63+
" [-l num] length of dimensions X and Y each local variable (default: %d)\n"
64+
" [-g num] gap at the end of each dimension (default: %d)\n"
65+
" -f file_name: output file name\n";
66+
fprintf(stderr, help, argv0, NVARS, LEN, GAP);
67+
}
68+
69+
/*----< main() >------------------------------------------------------------*/
70+
int main(int argc, char **argv)
71+
{
72+
char filename[512];
73+
size_t i, buf_len;
74+
int ret, err, rank, verbose, omode, nprocs, do_read, do_write, type_size;
75+
int nvars, len, gap, psize[2], gsize[2], count[2], start[2];
76+
char *buf;
77+
MPI_File fh;
78+
MPI_Datatype subType, filetype, buftype;
79+
MPI_Status status;
80+
MPI_Offset fsize;
81+
int *array_of_blocklengths;
82+
MPI_Aint lb, extent, *array_of_displacements;
83+
MPI_Datatype *array_of_types;
84+
MPI_Request req[2];
85+
86+
MPI_Init(&argc, &argv);
87+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
88+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
89+
90+
/* default values */
91+
len = LEN;
92+
gap = GAP;
93+
nvars = NVARS;
94+
do_write = 1;
95+
do_read = 1;
96+
verbose = 0;
97+
filename[0] = '\0';
98+
99+
/* get command-line arguments */
100+
while ((ret = getopt(argc, argv, "hvwrn:l:g:f:")) != EOF)
101+
switch(ret) {
102+
case 'v': verbose = 1;
103+
break;
104+
case 'w': do_read = 0;
105+
break;
106+
case 'r': do_write = 0;
107+
break;
108+
case 'n': nvars = atoi(optarg);
109+
break;
110+
case 'l': len = atoi(optarg);
111+
break;
112+
case 'g': gap = atoi(optarg);
113+
break;
114+
case 'f': strcpy(filename, optarg);
115+
break;
116+
case 'h':
117+
default: if (rank==0) usage(argv[0]);
118+
MPI_Finalize();
119+
return 1;
120+
}
121+
122+
if (filename[0] == '\0') {
123+
if (rank==0) usage(argv[0]);
124+
MPI_Finalize();
125+
return 1;
126+
}
127+
128+
array_of_blocklengths = (int*) malloc(sizeof(int) * nvars);
129+
array_of_displacements = (MPI_Aint*) malloc(sizeof(MPI_Aint) * nvars);
130+
array_of_types = (MPI_Datatype*) malloc(sizeof(MPI_Datatype) * nvars);
131+
132+
/* Creates a division of processors in a cartesian grid */
133+
psize[0] = psize[1] = 0;
134+
err = MPI_Dims_create(nprocs, 2, psize);
135+
CHECK_MPI_ERROR("MPI_Dims_create");
136+
137+
/* each 2D variable is of size gsizes[0] x gsizes[1] bytes */
138+
gsize[0] = len * psize[0];
139+
gsize[1] = len * psize[1];
140+
141+
/* set subarray offset and length */
142+
start[0] = len * (rank / psize[1]);
143+
start[1] = len * (rank % psize[1]);
144+
count[0] = len - gap; /* -1 to create holes */
145+
count[1] = len - gap;
146+
147+
fsize = (MPI_Offset)gsize[0] * gsize[1] * nvars - (len+1);
148+
if (verbose) {
149+
buf_len = (size_t)nvars * (len-1) * (len-1);
150+
if (rank == 0) {
151+
printf("Output file name = %s\n", filename);
152+
printf("nprocs=%d nvars=%d len=%d\n", nprocs, nvars, len);
153+
printf("Expecting file size=%lld bytes (%.1f MB, %.1f GB)\n",
154+
fsize*2, (float)fsize*2/1048576,(float)fsize*2/1073741824);
155+
printf("Each global variable is of size %d bytes (%.1f MB)\n",
156+
gsize[0]*gsize[1],(float)gsize[0]*gsize[1]/1048576);
157+
printf("Each process writes %zd bytes (%.1f MB, %.1f GB)\n",
158+
buf_len,(float)buf_len/1048576,(float)buf_len/1073741824);
159+
printf("** For nonblocking I/O test, the amount is twice\n");
160+
printf("-------------------------------------------------------\n");
161+
}
162+
printf("rank %3d: gsize=%4d %4d start=%4d %4d count=%4d %4d\n", rank,
163+
gsize[0],gsize[1],start[0],start[1],count[0],count[1]);
164+
}
165+
166+
/* create 2D subarray datatype for fileview */
167+
err = MPI_Type_create_subarray(2, gsize, count, start, MPI_ORDER_C, MPI_BYTE, &subType);
168+
CHECK_MPI_ERROR("MPI_Type_create_subarray");
169+
err = MPI_Type_commit(&subType);
170+
CHECK_MPI_ERROR("MPI_Type_commit");
171+
172+
/* create a filetype by concatenating nvars subType */
173+
for (i=0; i<nvars; i++) {
174+
array_of_blocklengths[i] = 1;
175+
array_of_displacements[i] = gsize[0]*gsize[1]*i;
176+
array_of_types[i] = subType;
177+
}
178+
err = MPI_Type_create_struct(nvars, array_of_blocklengths,
179+
array_of_displacements,
180+
array_of_types,
181+
&filetype);
182+
CHECK_MPI_ERROR("MPI_Type_create_struct");
183+
err = MPI_Type_commit(&filetype);
184+
CHECK_MPI_ERROR("MPI_Type_commit");
185+
err = MPI_Type_free(&subType);
186+
CHECK_MPI_ERROR("MPI_Type_free");
187+
188+
MPI_Type_size(filetype, &type_size);
189+
lb = 0;
190+
MPI_Type_get_extent(filetype, &lb, &extent);
191+
if (verbose && rank == 0)
192+
printf("file type size = %d extent = %ld\n", type_size, extent);
193+
194+
/* Create local buffer datatype: each 2D variable is of size len x len */
195+
gsize[0] = len;
196+
gsize[1] = len;
197+
start[0] = 0;
198+
start[1] = 0;
199+
count[0] = len - gap; /* -1 to create holes */
200+
count[1] = len - gap;
201+
202+
err = MPI_Type_create_subarray(2, gsize, count, start, MPI_ORDER_C, MPI_BYTE, &subType);
203+
CHECK_MPI_ERROR("MPI_Type_create_subarray");
204+
err = MPI_Type_commit(&subType);
205+
CHECK_MPI_ERROR("MPI_Type_commit");
206+
207+
/* concatenate nvars subType into a buftype */
208+
for (i=0; i<nvars; i++) {
209+
array_of_blocklengths[i] = 1;
210+
array_of_displacements[i] = len*len*i;
211+
array_of_types[i] = subType;
212+
}
213+
214+
/* create a buftype by concatenating nvars subTypes */
215+
err = MPI_Type_create_struct(nvars, array_of_blocklengths,
216+
array_of_displacements,
217+
array_of_types,
218+
&buftype);
219+
CHECK_MPI_ERROR("MPI_Type_create_struct");
220+
err = MPI_Type_commit(&buftype);
221+
CHECK_MPI_ERROR("MPI_Type_commit");
222+
err = MPI_Type_free(&subType);
223+
CHECK_MPI_ERROR("MPI_Type_free");
224+
225+
free(array_of_blocklengths);
226+
free(array_of_displacements);
227+
free(array_of_types);
228+
229+
MPI_Type_size(buftype, &type_size);
230+
lb = 0;
231+
MPI_Type_get_extent(buftype, &lb, &extent);
232+
if (verbose && rank == 0)
233+
printf("buffer type size = %d extent = %ld\n", type_size, extent);
234+
235+
/* allocate a local buffer */
236+
buf_len = (size_t)nvars * len * len;
237+
buf = (char*) malloc(buf_len);
238+
for (i=0; i<buf_len; i++) buf[i] = (char)rank;
239+
240+
/* open to create a file */
241+
omode = MPI_MODE_CREATE | MPI_MODE_RDWR;
242+
err = MPI_File_open(MPI_COMM_WORLD, filename, omode, MPI_INFO_NULL, &fh);
243+
CHECK_MPIO_ERROR("MPI_File_open");
244+
245+
/* set the file view */
246+
err = MPI_File_set_view(fh, 0, MPI_BYTE, filetype, "native", MPI_INFO_NULL);
247+
CHECK_MPIO_ERROR("MPI_File_set_view");
248+
249+
if (do_write) {
250+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
251+
CHECK_MPIO_ERROR("MPI_File_seek");
252+
253+
/* MPI collective write */
254+
err = MPI_File_write_all(fh, buf, 1, buftype, &status);
255+
CHECK_MPIO_ERROR("MPI_File_write_all");
256+
257+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
258+
CHECK_MPIO_ERROR("MPI_File_seek");
259+
260+
/* MPI independent write */
261+
err = MPI_File_write(fh, buf, 1, buftype, &status);
262+
CHECK_MPIO_ERROR("MPI_File_write");
263+
264+
/* MPI nonblocking collective write */
265+
char *buf2 = (char*) malloc(buf_len);
266+
for (i=0; i<buf_len; i++) buf2[i] = (char)rank;
267+
268+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
269+
CHECK_MPIO_ERROR("MPI_File_seek");
270+
271+
err = MPI_File_iwrite_all(fh, buf, 1, buftype, &req[0]);
272+
CHECK_MPIO_ERROR("MPI_File_iwrite_all 1");
273+
274+
err = MPI_File_iwrite_all(fh, buf2, 1, buftype, &req[1]);
275+
CHECK_MPIO_ERROR("MPI_File_iwrite_all 2");
276+
277+
err = MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
278+
CHECK_MPIO_ERROR("MPI_Waitall");
279+
280+
/* MPI nonblocking independent write */
281+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
282+
CHECK_MPIO_ERROR("MPI_File_seek");
283+
284+
err = MPI_File_iwrite(fh, buf, 1, buftype, &req[0]);
285+
CHECK_MPIO_ERROR("MPI_File_iwrite 1");
286+
287+
err = MPI_File_iwrite(fh, buf2, 1, buftype, &req[1]);
288+
CHECK_MPIO_ERROR("MPI_File_iwrite 2");
289+
290+
err = MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
291+
CHECK_MPIO_ERROR("MPI_Waitall");
292+
293+
free(buf2);
294+
}
295+
296+
if (do_read) {
297+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
298+
CHECK_MPIO_ERROR("MPI_File_seek");
299+
300+
/* MPI collective read */
301+
err = MPI_File_read_all(fh, buf, 1, buftype, &status);
302+
CHECK_MPIO_ERROR("MPI_File_read_all");
303+
304+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
305+
CHECK_MPIO_ERROR("MPI_File_seek");
306+
307+
/* MPI independent read */
308+
err = MPI_File_read(fh, buf, 1, buftype, &status);
309+
CHECK_MPIO_ERROR("MPI_File_read");
310+
311+
/* MPI nonblocking collective read */
312+
char *buf2 = (char*) malloc(buf_len);
313+
for (i=0; i<buf_len; i++) buf2[i] = (char)rank;
314+
315+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
316+
CHECK_MPIO_ERROR("MPI_File_seek");
317+
318+
err = MPI_File_iread_all(fh, buf, 1, buftype, &req[0]);
319+
CHECK_MPIO_ERROR("MPI_File_iread_all 1");
320+
321+
err = MPI_File_iread_all(fh, buf2, 1, buftype, &req[1]);
322+
CHECK_MPIO_ERROR("MPI_File_iread_all 2");
323+
324+
err = MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
325+
CHECK_MPIO_ERROR("MPI_Waitall");
326+
327+
/* MPI nonblocking independent read */
328+
err = MPI_File_seek(fh, 0, MPI_SEEK_SET);
329+
CHECK_MPIO_ERROR("MPI_File_seek");
330+
331+
err = MPI_File_iread(fh, buf, 1, buftype, &req[0]);
332+
CHECK_MPIO_ERROR("MPI_File_iread 1");
333+
334+
err = MPI_File_iread(fh, buf2, 1, buftype, &req[1]);
335+
CHECK_MPIO_ERROR("MPI_File_iread 2");
336+
337+
err = MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
338+
CHECK_MPIO_ERROR("MPI_Waitall");
339+
340+
free(buf2);
341+
}
342+
343+
err = MPI_File_close(&fh);
344+
CHECK_MPIO_ERROR("MPI_File_close");
345+
346+
free(buf);
347+
348+
err = MPI_Type_free(&filetype);
349+
CHECK_MPI_ERROR("MPI_Type_free");
350+
err = MPI_Type_free(&buftype);
351+
CHECK_MPI_ERROR("MPI_Type_free");
352+
353+
MPI_Finalize();
354+
return 0;
355+
}

0 commit comments

Comments
 (0)