Derecho: updated spack-stack, job_card, RT tests configurations

Author: Natalie Perlin

modulefiles/ufs_derecho.intel.lua

@@ -3,11 +3,11 @@ loads UFS Model prerequisites for NOAA Parallelworks/Intel
 ]])
 
 setenv("LMOD_TMOD_FIND_FIRST","yes")
-prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra")
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/modulefiles/cray-mpich/8.1.29-4natrhl/gcc/12.2.0")
+prepend_path("MODULEPATH", "/glade/derecho/scratch/nperlin/spack-stack-R1.9.0/envs/ue-oneapi-2024.2.1/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/glade/derecho/scratch/nperlin/spack-stack-R1.9.0/envs/ue-oneapi-2024.2.1/install/modulefiles/cray-mpich/8.1.29-56ro2ht/gcc/12.4.0")
 
-unload("ncarcompilers")
+-- unload("ncarcompilers")
+prepend_path("MODULEPATH", "/glade/derecho/scratch/nperlin/UFS-WM/ufs-weather-model/modulefiles/stack")
 stack_intel_ver=os.getenv("stack_intel_ver") or "2024.2.1"
 load(pathJoin("stack-oneapi", stack_intel_ver))
 
@@ -21,6 +21,7 @@ stack_python_ver=os.getenv("stack_python_ver") or "3.11.7"
 load(pathJoin("stack-python", stack_python_ver))
 
 setenv("CMAKE_Platform", "derecho.intel")
+
 load("ufs-weather-model-env")
 load("sp/2.5.0")
 load("crtm/2.4.0.1")

tests/fv3_conf/fv3_qsub.IN_derecho

@@ -4,7 +4,7 @@
 #PBS -N @[JBNME]
 #PBS -A @[ACCNR]
 #PBS -q @[QUEUE]
-#PBS -l select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]:ompthreads=@[THRD]
+#PBS -l select=@[NODES]:ncpus=@[TPN]:ompthreads=@[THRD]
 #PBS -l walltime=00:@[WLCLK]:00
 
 set -eux
@@ -30,6 +30,10 @@ export OMP_NUM_THREADS=@[THRD]
 export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
 export ESMF_RUNTIME_PROFILE=ON
 export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+export I_MPI_EXTRA_FILESYSTEM="ON"
+
+export FI_CXI_RX_MATCH_MODE=hybrid
+export FI_MR_HMEM=0
 
 # Avoid job errors because of filesystem synchronization delays
 sync && sleep 1
@@ -41,7 +45,7 @@ if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then
     false
 fi
 
-mpiexec -n @[TASKS] -ppn @[TPN] -depth @[THRD] ./fv3.exe
+mpiexec -n @[UFS_TASKS] -depth @[THRD] ./fv3.exe
 
 echo "Model ended:    " `date`
 echo -n " $( date +%s )," >> job_timestamp.txt

tests/tests/cpld_control_c192_p8

@@ -140,9 +140,6 @@ export FNABSC="'C192.maximum_snow_albedo.tileX.nc'"
 
 export FV3_RUN=cpld_control_run.IN
 
-if [[ $MACHINE_ID = derecho ]]; then
-  TPN=96
-fi
 if [[ $MACHINE_ID = gaeac6 ]]; then
   TPN=144
 fi

tests/tests/cpld_control_ciceC_p8

@@ -94,6 +94,6 @@ export N_SPLIT=5
 export DO_GWD_OPT_PSL=.true.
 export DO_GSL_DRAG_SS=.false.
 
-if [[ $MACHINE_ID = gaeac6 ]] || [[ $MACHINE_ID = derecho ]]; then
+if [[ $MACHINE_ID = gaeac6 ]]; then
   TPN=96
 fi

tests/tests/cpld_control_p8.v2.sfc

@@ -87,7 +87,7 @@ export eps_imesh=2.5e-1
 export FV3_RUN=cpld_control_run.IN
 export V2_SFC_FILE=true
 
-if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then
+if [[ $MACHINE_ID = gaeac6 ]]; then
   TPN=96
 fi
 

tests/tests/cpld_decomp_p8

@@ -87,7 +87,7 @@ export eps_imesh=2.5e-1
 
 export FV3_RUN=cpld_control_run.IN
 
-if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then
+if [[ $MACHINE_ID = gaeac6 ]]; then
   TPN=96
 fi
 

tests/tests/cpld_mpi_p8

@@ -87,7 +87,7 @@ export eps_imesh=2.5e-1
 
 export FV3_RUN=cpld_control_run.IN
 
-if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then
+if [[ $MACHINE_ID = gaeac6 ]]; then
   TPN=96
 fi
 

tests/tests/cpld_restart_p8

@@ -93,7 +93,7 @@ export CICE_DIAGFREQ=`expr $((FHMAX-$FHROT)) \* 3600 / $DT_CICE`
 
 export FV3_RUN=cpld_control_run.IN
 
-if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then
+if [[ $MACHINE_ID = gaeac6 ]]; then
   TPN=96
 fi
 

tests/tests/hafs_regional_storm_following_1nest_atm_ocn_wav

@@ -143,3 +143,4 @@ export INPUT_NEST02_NML=input_nest_hafs.nml.IN
 export MODEL_CONFIGURE=model_configure_hafs.IN
 export UFS_CONFIGURE=ufs.configure.hafs_atm_ocn_wav.IN
 export FV3_RUN="hafs_fv3_run.IN hycom_hat10_run.IN hafs_ww3_run.IN"
+