Skip to content

Commit 8321cdd

Browse files
BinLiu-NOAABinLiu-NOAA
authored
Merge pull request #139 from hafs-community/feature/hafs.v0.3.0
Bring HAFSv0.3.0 series developments back to HAFS develop branch
2 parents 3fd3bc5 + 6497dcc commit 8321cdd

File tree

140 files changed

+6689
-817
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

140 files changed

+6689
-817
lines changed

‎.gitmodules

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[submodule "ufs-weather-model"]
22
path = sorc/hafs_forecast.fd
3-
url = https://github.com/ufs-community/ufs-weather-model.git
4-
branch = develop
3+
url = https://github.com/hafs-community/ufs-weather-model.git
4+
branch = support/HAFS
55
[submodule "UFS_UTILS"]
66
path = sorc/hafs_utils.fd
77
url = https://github.com/hafs-community/UFS_UTILS.git

‎jobs/JHAFS_ANALYSIS

Lines changed: 128 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20-
module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21-
module load modulefile.ProdGSI.${machine}
20+
#module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21+
#module load modulefile.ProdGSI.${machine}
2222
module list
2323
fi
2424

@@ -42,9 +42,44 @@ export ENSDA=${ENSDA:-NO}
4242
export GSI_D01=${GSI_D01:-NO}
4343
export GSI_D02=${GSI_D02:-NO}
4444

45+
yr=`echo $CDATE | cut -c1-4`
46+
mn=`echo $CDATE | cut -c5-6`
47+
dy=`echo $CDATE | cut -c7-8`
48+
49+
export NDATE=${NDATE:-ndate}
50+
export NCP=${NCP:-"/bin/cp"}
51+
CDATEprior=`${NDATE} -6 $CDATE`
52+
export COMhafsprior=${COMhafsprior:-${COMhafs}/../../${CDATEprior}/${STORMID}}
53+
54+
if [ ${RUN_ATM_VI_FGAT} = "YES" ]; then
55+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_vi_fgat06
56+
elif [ ${RUN_GSI_VR_FGAT} = "YES" ]; then
57+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_analysis_vr_fgat06
58+
elif [ ${RUN_ATM_MERGE_FGAT} = "YES" ]; then
59+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_merge_fgat06
60+
elif [ ${RUN_ATM_INIT_FGAT} = "YES" ]; then
61+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_init_fgat06
62+
else
63+
if [ ${RUN_ATM_VI} = "YES" ]; then
64+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_vi
65+
elif [ ${RUN_GSI_VR} = "YES" ]; then
66+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_analysis_vr
67+
elif [ ${RUN_ATM_MERGE} = "YES" ]; then
68+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_merge
69+
elif [ ${RUN_ATM_INIT} = "YES" ]; then
70+
RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_init
71+
else
72+
RESTARTinp_fgat06=${COMhafsprior}/RESTART
73+
fi
74+
fi
75+
RESTARTinp=${RESTARTinp_fgat06}
76+
4577
# analysis for d01
4678
if [ ${GSI_D01} = YES ]; then
4779

80+
export neststr=${neststr:-""}
81+
export tilestr=${tilestr:-".tile1"}
82+
export nesttilestr=${nesttilestr:-""}
4883
export DATA=${WORKhafs}/analysis
4984

5085
export SCRUBDATA=${SCRUBDATA:-YES}
@@ -63,10 +98,55 @@ if [ "${KEEPDATA^^}" != YES ]; then
6398
rm -rf $DATA
6499
fi
65100

101+
else
102+
103+
export neststr=${neststr:-""}
104+
export tilestr=${tilestr:-".tile1"}
105+
export nesttilestr=${nesttilestr:-""}
106+
107+
export RESTARTanl=${RESTARTanl:-${WORKhafs}/intercom/RESTART_analysis}
108+
mkdir -p ${RESTARTanl}
109+
110+
if [ -s ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ]; then
111+
112+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.coupler.res ${RESTARTanl}/${PDY}.${cyc}0000.coupler.res
113+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc
114+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc
115+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc
116+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc
117+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc
118+
119+
${NCP} ${RESTARTinp}/oro_data${nesttilestr}.nc ${RESTARTanl}/oro_data${nesttilestr}.nc
120+
${NCP} ${RESTARTinp}/atmos_static${nesttilestr}.nc ${RESTARTanl}/atmos_static${nesttilestr}.nc
121+
${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ${RESTARTanl}/grid_spec${nesttilestr}.nc
122+
123+
# pass over phy_data as well
124+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc
125+
126+
if [[ ! -z "$neststr" ]] ; then
127+
if [ -e ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ]; then
128+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc
129+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc
130+
fi
131+
fi
132+
133+
# Pass over the grid_mspec files for moving nest
134+
if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then
135+
if [[ -z "$neststr" ]] && [[ $tilestr = ".tile1" ]]; then
136+
# "grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc" for domain 02
137+
${NCP} -p ${RESTARTinp}/grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc ${RESTARTanl}/
138+
else
139+
# "grid_mspec.nest02_${yr}_${mn}_${dy}_${cyc}.tile2.nc" for domain 02
140+
${NCP} -p ${RESTARTinp}/grid_mspec${neststr}_${yr}_${mn}_${dy}_${cyc}${tilestr}.nc ${RESTARTanl}/
141+
fi
142+
fi
143+
144+
fi
145+
66146
fi
67147

68148
# analysis for d02
69-
if [ ${GSI_D02} = YES ]; then
149+
if [ ${GSI_D02} = YES ] && [[ $nest_grids -ge 2 ]]; then
70150

71151
export neststr=".nest02"
72152
export tilestr=".tile2"
@@ -89,6 +169,51 @@ if [ "${KEEPDATA^^}" != YES ]; then
89169
rm -rf $DATA
90170
fi
91171

172+
elif [ ${GSI_D02} != YES ] && [[ $nest_grids -ge 2 ]]; then
173+
174+
export neststr=".nest02"
175+
export tilestr=".tile2"
176+
export nesttilestr=".nest02.tile2"
177+
178+
export RESTARTanl=${RESTARTanl:-${WORKhafs}/intercom/RESTART_analysis}
179+
mkdir -p ${RESTARTanl}
180+
181+
if [ -s ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ]; then
182+
183+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.coupler.res ${RESTARTanl}/${PDY}.${cyc}0000.coupler.res
184+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc
185+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc
186+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc
187+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc
188+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc
189+
190+
${NCP} ${RESTARTinp}/oro_data${nesttilestr}.nc ${RESTARTanl}/oro_data${nesttilestr}.nc
191+
${NCP} ${RESTARTinp}/atmos_static${nesttilestr}.nc ${RESTARTanl}/atmos_static${nesttilestr}.nc
192+
${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ${RESTARTanl}/grid_spec${nesttilestr}.nc
193+
194+
# pass over phy_data as well
195+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc
196+
197+
if [[ ! -z "$neststr" ]] ; then
198+
if [ -e ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ]; then
199+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc
200+
${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc
201+
fi
202+
fi
203+
204+
# Pass over the grid_mspec files for moving nest
205+
if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then
206+
if [[ -z "$neststr" ]] && [[ $tilestr = ".tile1" ]]; then
207+
# "grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc" for domain 02
208+
${NCP} -p ${RESTARTinp}/grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc ${RESTARTanl}/
209+
else
210+
# "grid_mspec.nest02_${yr}_${mn}_${dy}_${cyc}.tile2.nc" for domain 02
211+
${NCP} -p ${RESTARTinp}/grid_mspec${neststr}_${yr}_${mn}_${dy}_${cyc}${tilestr}.nc ${RESTARTanl}/
212+
fi
213+
fi
214+
215+
fi
216+
92217
fi
93218

94219
date

‎jobs/JHAFS_ANALYSIS_VR

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20-
module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21-
module load modulefile.ProdGSI.${machine}
20+
#module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21+
#module load modulefile.ProdGSI.${machine}
2222
module list
2323
fi
2424

‎jobs/JHAFS_ATM_IC

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20+
if [[ ${machine} != "wcoss2" ]]; then
21+
module purge
22+
fi
23+
#module unload module load mapl/2.11.0-esmf-8.3.0b09
2024
module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles
2125
module load build.${machine}.intel
2226
module list

‎jobs/JHAFS_ATM_INIT

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ export FGAT_MODEL=${FGAT_MODEL:-gfs}
5151
export FGAT_HR=${FGAT_HR:-00}
5252

5353
export run_init=yes
54+
5455
if [ ${FGAT_MODEL} = gdas ]; then
5556
export CDATE=$(${NDATE} $(awk "BEGIN {print ${FGAT_HR}-6}") $CDATE)
5657
# export out_prefix=$(echo "${STORM}${STORMID}.${CDATE}" | tr '[A-Z]' '[a-z]')
@@ -135,6 +136,10 @@ date
135136
#===============================================================================
136137
# post
137138

139+
export TOTAL_TASKS=120
140+
export OMP_NUM_THREADS=1
141+
source ${USHhafs}/hafs_runcmd.sh.inc
142+
138143
if [ ${ENSDA} = YES ]; then
139144
export ENSID=${ENSID:-001}
140145
export INPdir=${WORKhafs}/atm_init_ens/mem${ENSID}/forecast
@@ -249,8 +254,15 @@ fi
249254
for ng in $(seq 1 ${ngrids});
250255
do
251256

257+
if [[ $ng -eq 1 ]]; then
258+
neststr=""
259+
else
260+
neststr=".nest$(printf '%02d' ${ng})"
261+
fi
252262
gridstr=".grid$(printf '%02d' ${ng})"
253-
if grep "PROGRAM GETTRK HAS ENDED" ./run_product${gridstr}.log ; then
263+
cat ./run_product${gridstr}.log
264+
#if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then
265+
if grep "top of output_all" ./tracker${neststr}/hafs_gettrk.out ; then
254266
echo "INFO: successfully ran run_product${gridstr}"
255267
else
256268
echo "ERROR: failed running run_product${gridstr}"

‎jobs/JHAFS_ATM_LBC

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20+
if [[ ${machine} != "wcoss2" ]]; then
21+
module purge
22+
fi
2023
module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles
2124
module load build.${machine}.intel
2225
module list

‎jobs/JHAFS_ATM_PREP

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20+
if [ ${machine} != wcoss2 ]; then
21+
module purge
22+
fi
2023
module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles
2124
module load build.${machine}.intel
2225
module list
@@ -39,6 +42,9 @@ export cyc=${cyc:?}
3942
export STORM=${STORM:-FAKE}
4043
export STORMID=${STORMID:-00L}
4144
export ENSDA=${ENSDA:-NO}
45+
export ATM_PREP_MVNEST=${ATM_PREP_MVNEST:-NO}
46+
47+
if [[ ${ATM_PREP_MVNEST} != YES ]]; then
4248

4349
# Deterministic or ensemble
4450
if [ ${ENSDA} = YES ]; then
@@ -86,6 +92,8 @@ if [ "${KEEPDATA^^}" != YES ]; then
8692
rm -rf $DATA
8793
fi
8894

95+
elif [[ ${ATM_PREP_MVNEST} = YES ]]; then
96+
8997
# If moving nest is enabled
9098
if [[ "${is_moving_nest}" = *".true."* ]] || [[ "${is_moving_nest}" = *".T."* ]] ; then
9199

@@ -126,4 +134,6 @@ fi
126134

127135
fi
128136

137+
fi
138+
129139
date

‎jobs/JHAFS_EMCGRAPHICS

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ export USHgraph=${USHgraph:-${HOMEgraph}/ush}
3939
export WORKgraph=${WORKgraph:-${COMhafs}/../../../${YMDH}/${STORMID}/emc_graphics}
4040
export COMgraph=${COMgraph:-${COMhafs}/emc_graphics}
4141

42+
export MPISERIAL=${MPISERIAL:-${EXEChafs}/hafs_mpiserial.x}
43+
4244
export ADECKgraph=${ADECKhafs:?}
4345
export BDECKgraph=${BDECKhafs:?}
4446
export SYNDAThafs=${SYNDAThafs:?}

‎jobs/JHAFS_ENKF_HX

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20-
module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21-
module load modulefile.ProdGSI.${machine}
20+
#module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21+
#module load modulefile.ProdGSI.${machine}
2222
module list
2323
fi
2424

‎jobs/JHAFS_ENKF_MEAN

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
1717
export envir=${envir:-prod} # prod, para, test
1818
export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
1919
if [ "${RUN_ENVIR^^}" != NCO ]; then
20-
module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21-
module load modulefile.ProdGSI.${machine}
20+
#module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
21+
#module load modulefile.ProdGSI.${machine}
2222
module list
2323
fi
2424

0 commit comments

Comments
 (0)