From b4352409b890093a34ab600915543a02bbd811be Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 30 Jul 2024 20:39:22 -0400 Subject: [PATCH 1/5] small mods to v3 script --- run_e3sm.template.sh | 59 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 623867c01904..1a55dc933d87 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -5,9 +5,6 @@ # Bash coding style inspired by: # http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming -# TO DO: -# - custom pelayout - main() { # For debugging, uncomment line below @@ -46,8 +43,8 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" -readonly CASE_ROOT="/pscratch/sd/r/${USER}/e3sm-scratch/${CASE_NAME}" +readonly CODE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/your_casename/code/${CHECKOUT}" +readonly CASE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/${CASE_NAME}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build @@ -56,6 +53,7 @@ readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive # Define type of run # short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', # 'M_1x10_ndays', 'M2_1x10_ndays', 'M80_1x10_ndays', 'L_1x10_ndays' +# * can replace XS, M, etc with custom-XY with XY being the node count # or 'production' for full simulation readonly run='XS_2x5_ndays' if [ "${run}" != "production" ]; then @@ -118,6 +116,9 @@ fetch_code # Create case create_newcase +# Custom PE layout +custom_pelayout + # Setup case_setup @@ -322,6 +323,14 @@ create_newcase() { echo $'\n----- Starting create_newcase -----\n' + + if [[ ${PELAYOUT} == custom-* ]]; + then + layout="M" # temporary placeholder for create_newcase + else + layout=${PELAYOUT} + fi + if [[ -z "$CASE_GROUP" ]]; then ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ @@ -333,7 +342,7 @@ create_newcase() { --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} \ - --pecount ${PELAYOUT} + --pecount ${layout} else ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ @@ -346,7 +355,7 @@ create_newcase() { --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} \ - --pecount ${PELAYOUT} + --pecount ${layout} fi @@ -398,6 +407,37 @@ case_setup() { popd } +#----------------------------------------------------- +custom_pelayout() { + +if [[ ${PELAYOUT} == custom-* ]]; +then + echo $'\n CUSTOMIZE PROCESSOR CONFIGURATION:' + + # Number of cores per node (machine specific) + if [ "${MACHINE}" == "pm-cpu" ]; then + ncore=128 + else + echo 'ERROR: MACHINE = '${MACHINE}' is not supported for custom PE layout.' + exit 400 + fi + + # Extract number of nodes + tmp=($(echo ${PELAYOUT} | tr "-" " ")) + nnodes=${tmp[1]} + + # Customize + pushd ${CASE_SCRIPTS_DIR} + ./xmlchange NTASKS=$(( $nnodes * $ncore )) + ./xmlchange NTHRDS=1 + ./xmlchange MAX_MPITASKS_PER_NODE=$ncore + ./xmlchange MAX_TASKS_PER_NODE=$ncore + popd + +fi + +} + #----------------------------------------------------- case_build() { @@ -534,9 +574,10 @@ copy_script() { local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance mkdir -p ${script_provenance_dir} - local this_script_name=`basename $0` + local this_script_name=$( basename -- "$0"; ) + local this_script_dir=$( dirname -- "$0"; ) local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` - cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + cp -vp "${this_script_dir}/${this_script_name}" ${script_provenance_dir}/${script_provenance_name} } From 33b4eaa5c04b7f1a8e866c5ef4b174e1c8016b1d Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Tue, 30 Jul 2024 20:44:30 -0400 Subject: [PATCH 2/5] Use $PSCRATCH instead --- run_e3sm.template.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 1a55dc933d87..e2a5b603ba79 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -43,8 +43,8 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CODE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/your_casename/code/${CHECKOUT}" -readonly CASE_ROOT="/pscratch/sd/${USER:0:1}/${USER}/e3sm-scratch/${MACHINE}/${CASE_NAME}" +readonly CASE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}" +readonly CODE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}/code/${CHECKOUT}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build From 7581ae045f308b6544e61f9c30db20536441a27a Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Mon, 5 Aug 2024 09:19:07 -0400 Subject: [PATCH 3/5] fix link to best practices, save code in home --- run_e3sm.template.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index e2a5b603ba79..413f2bf75286 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -3,7 +3,7 @@ # E3SM Coupled Model Group run_e3sm script template. # # Bash coding style inspired by: -# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming +# https://web.archive.org/web/20200620202413/http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming main() { @@ -44,7 +44,7 @@ readonly GET_REFCASE=TRUE # Set paths readonly CASE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}" -readonly CODE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}/code/${CHECKOUT}" +readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build From fd620a5c8779956548980e335498ffef16552581 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Wed, 21 Aug 2024 09:37:50 -0400 Subject: [PATCH 4/5] gen custom_pelayout, better case loc, and default proj --- run_e3sm.template.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 413f2bf75286..86e360157598 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -15,7 +15,7 @@ main() { # Machine and project readonly MACHINE=pm-cpu # BEFORE RUNNING: CHANGE this to your project -readonly PROJECT="e3sm" +readonly PROJECT="$(sacctmgr show user $USER format=DefaultAccount | tail -n1 | tr -d ' ')" # Simulation readonly COMPSET="WCYCL1850" @@ -43,7 +43,7 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths -readonly CASE_ROOT="${PSCRATCH}/e3sm-scratch/${MACHINE}/${CASE_NAME}" +readonly CASE_ROOT="${PSCRATCH}/E3SMv3/${CASE_NAME}" readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" # Sub-directories @@ -417,6 +417,12 @@ then # Number of cores per node (machine specific) if [ "${MACHINE}" == "pm-cpu" ]; then ncore=128 + elif [ "${MACHINE}" == "chrysalis" ]; then + ncore=64 + elif [ "${MACHINE}" == "compy" ]; then + ncore=40 + elif [ "${MACHINE}" == "anvil" ]; then + ncore=36 else echo 'ERROR: MACHINE = '${MACHINE}' is not supported for custom PE layout.' exit 400 From f84cad1e7e817b248ebfe45f7270cb36db29b8f5 Mon Sep 17 00:00:00 2001 From: Naser Mahfouz Date: Fri, 23 Aug 2024 10:15:38 -0400 Subject: [PATCH 5/5] improve the comment regarding PROJECT --- run_e3sm.template.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 86e360157598..49869f5c6f63 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -14,7 +14,8 @@ main() { # Machine and project readonly MACHINE=pm-cpu -# BEFORE RUNNING: CHANGE this to your project +# NOTE: The command below will return your default project on SLURM-based systems. +# If you are not using SLURM or need a different project, remove the command and set it directly readonly PROJECT="$(sacctmgr show user $USER format=DefaultAccount | tail -n1 | tr -d ' ')" # Simulation @@ -53,7 +54,7 @@ readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive # Define type of run # short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', # 'M_1x10_ndays', 'M2_1x10_ndays', 'M80_1x10_ndays', 'L_1x10_ndays' -# * can replace XS, M, etc with custom-XY with XY being the node count +# * can replace XS, M, etc. with custom-XY with XY being the node count # or 'production' for full simulation readonly run='XS_2x5_ndays' if [ "${run}" != "production" ]; then