diff --git a/run_e3sm.template.sh b/run_e3sm.template.sh index 623867c01904..49869f5c6f63 100755 --- a/run_e3sm.template.sh +++ b/run_e3sm.template.sh @@ -3,10 +3,7 @@ # E3SM Coupled Model Group run_e3sm script template. # # Bash coding style inspired by: -# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming - -# TO DO: -# - custom pelayout +# https://web.archive.org/web/20200620202413/http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming main() { @@ -17,8 +14,9 @@ main() { # Machine and project readonly MACHINE=pm-cpu -# BEFORE RUNNING: CHANGE this to your project -readonly PROJECT="e3sm" +# NOTE: The command below will return your default project on SLURM-based systems. +# If you are not using SLURM or need a different project, remove the command and set it directly +readonly PROJECT="$(sacctmgr show user $USER format=DefaultAccount | tail -n1 | tr -d ' ')" # Simulation readonly COMPSET="WCYCL1850" @@ -46,8 +44,8 @@ readonly GET_REFCASE=TRUE #readonly RUN_REFDATE="" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' # Set paths +readonly CASE_ROOT="${PSCRATCH}/E3SMv3/${CASE_NAME}" readonly CODE_ROOT="${HOME}/E3SMv3/code/${CHECKOUT}" -readonly CASE_ROOT="/pscratch/sd/r/${USER}/e3sm-scratch/${CASE_NAME}" # Sub-directories readonly CASE_BUILD_DIR=${CASE_ROOT}/build @@ -56,6 +54,7 @@ readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive # Define type of run # short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', # 'M_1x10_ndays', 'M2_1x10_ndays', 'M80_1x10_ndays', 'L_1x10_ndays' +# * can replace XS, M, etc. with custom-XY with XY being the node count # or 'production' for full simulation readonly run='XS_2x5_ndays' if [ "${run}" != "production" ]; then @@ -118,6 +117,9 @@ fetch_code # Create case create_newcase +# Custom PE layout +custom_pelayout + # Setup case_setup @@ -322,6 +324,14 @@ create_newcase() { echo $'\n----- Starting create_newcase -----\n' + + if [[ ${PELAYOUT} == custom-* ]]; + then + layout="M" # temporary placeholder for create_newcase + else + layout=${PELAYOUT} + fi + if [[ -z "$CASE_GROUP" ]]; then ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ @@ -333,7 +343,7 @@ create_newcase() { --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} \ - --pecount ${PELAYOUT} + --pecount ${layout} else ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ @@ -346,7 +356,7 @@ create_newcase() { --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} \ - --pecount ${PELAYOUT} + --pecount ${layout} fi @@ -398,6 +408,43 @@ case_setup() { popd } +#----------------------------------------------------- +custom_pelayout() { + +if [[ ${PELAYOUT} == custom-* ]]; +then + echo $'\n CUSTOMIZE PROCESSOR CONFIGURATION:' + + # Number of cores per node (machine specific) + if [ "${MACHINE}" == "pm-cpu" ]; then + ncore=128 + elif [ "${MACHINE}" == "chrysalis" ]; then + ncore=64 + elif [ "${MACHINE}" == "compy" ]; then + ncore=40 + elif [ "${MACHINE}" == "anvil" ]; then + ncore=36 + else + echo 'ERROR: MACHINE = '${MACHINE}' is not supported for custom PE layout.' + exit 400 + fi + + # Extract number of nodes + tmp=($(echo ${PELAYOUT} | tr "-" " ")) + nnodes=${tmp[1]} + + # Customize + pushd ${CASE_SCRIPTS_DIR} + ./xmlchange NTASKS=$(( $nnodes * $ncore )) + ./xmlchange NTHRDS=1 + ./xmlchange MAX_MPITASKS_PER_NODE=$ncore + ./xmlchange MAX_TASKS_PER_NODE=$ncore + popd + +fi + +} + #----------------------------------------------------- case_build() { @@ -534,9 +581,10 @@ copy_script() { local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance mkdir -p ${script_provenance_dir} - local this_script_name=`basename $0` + local this_script_name=$( basename -- "$0"; ) + local this_script_dir=$( dirname -- "$0"; ) local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` - cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + cp -vp "${this_script_dir}/${this_script_name}" ${script_provenance_dir}/${script_provenance_name} }