Add a patch to ROMIO to print the number of calls to memcpy()

This is to show the excessive number of calls to memcpy() in
macro ADIOI_BUF_COPY of subroutine ADIOI_Fill_send_buffer() in
file mpich/src/mpi/romio/adio/common/ad_write_coll.c

Run pio_noncontig with the following commands and compare.

% mpiexec -n 16 pio_noncontig -k 256 -c 32768 -w
Number of global variables = 64
Each global variable is of size 256 x 32768 bytes
Each  local variable is of size 256 x 16 bytes
Gap between the first 2 variables is of size 16 bytes
Number of subarray types concatenated is 8192
Each process makes a request of amount 33554688 bytes
ROMIO hint set: cb_buffer_size = 1048576
ROMIO hint set: cb_nodes = 4
rank 0: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153
rank 1: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153
rank 2: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153
rank 3: ADIOI_GEN_WriteStridedColl number of memcpy called = 2097153
---------------------------------------------------------
Time of collective write = 30.83 sec
---------------------------------------------------------

% mpiexec -n 16 pio_noncontig -k 256 -c 32768 -w -g 0
Number of global variables = 64
Each global variable is of size 256 x 32768 bytes
Each  local variable is of size 256 x 16 bytes
Gap between the first 2 variables is of size 0 bytes
Number of subarray types concatenated is 8192
Each process makes a request of amount 33554688 bytes
ROMIO hint set: cb_buffer_size = 1048576
ROMIO hint set: cb_nodes = 4
rank 0: ADIOI_GEN_WriteStridedColl number of memcpy called = 0
rank 1: ADIOI_GEN_WriteStridedColl number of memcpy called = 0
rank 2: ADIOI_GEN_WriteStridedColl number of memcpy called = 0
rank 3: ADIOI_GEN_WriteStridedColl number of memcpy called = 0
---------------------------------------------------------
Time of collective write = 9.27 sec
---------------------------------------------------------

Author: wkliao

tests/0001-print-number-of-calls-to-memcpy.patch

@@ -0,0 +1,50 @@
+From 82487c29f5737b8f4c959a8dfc148d52deed0c51 Mon Sep 17 00:00:00 2001
+From: wkliao <wkliao@users.noreply.github.com>
+Date: Thu, 18 Apr 2024 15:44:46 -0500
+Subject: [PATCH] print number of calls to memcpy()
+
+---
+ src/mpi/romio/adio/common/ad_write_coll.c | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/src/mpi/romio/adio/common/ad_write_coll.c b/src/mpi/romio/adio/common/ad_write_coll.c
+index 2b7ed69f..84c5875d 100644
+--- a/src/mpi/romio/adio/common/ad_write_coll.c
++++ b/src/mpi/romio/adio/common/ad_write_coll.c
+@@ -10,6 +10,8 @@
+ #include "mpe.h"
+ #endif
+ 
++static int num_memcpy;
++
+ /* prototypes of functions used for collective writes only. */
+ static void ADIOI_Exch_and_write(ADIO_File fd, void *buf, MPI_Datatype
+                                  datatype, int nprocs, int myrank,
+@@ -78,6 +80,7 @@ void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
+     ADIO_Offset *len_list = NULL;
+     int old_error, tmp_error;
+ 
++num_memcpy = 0;
+     if (fd->hints->cb_pfr != ADIOI_HINT_DISABLE) {
+         /* Cast away const'ness as the below function is used for read
+          * and write */
+@@ -247,6 +250,8 @@ void ADIOI_GEN_WriteStridedColl(ADIO_File fd, const void *buf, MPI_Aint count,
+ #ifdef AGGREGATION_PROFILE
+     MPE_Log_event(5013, 0, NULL);
+ #endif
++
++if (fd->is_agg) printf("rank %d: %s number of memcpy called = %d\n",myrank,__func__,num_memcpy);
+ }
+ 
+ 
+@@ -791,6 +796,7 @@ static void ADIOI_W_Exchange_data(ADIO_File fd, void *buf, char *write_buf,
+   ADIOI_Assert(size_in_buf == (size_t)size_in_buf); \
+         memcpy(&(send_buf[p][send_buf_idx[p]]), \
+                ((char *) buf) + user_buf_idx, size_in_buf); \
++num_memcpy++; \
+         send_buf_idx[p] += size_in_buf; \
+         user_buf_idx += size_in_buf; \
+         flat_buf_sz -= size_in_buf; \
+-- 
+2.39.3
+