Merge branch 'develop' into feature/prep_emis_rework

Author: bbakernoaa

.github/CODEOWNERS

@@ -74,7 +74,6 @@ jobs/JGLOBAL_MARINE_* @guillaumevernieres @AndrewEichmann-NOAA
 jobs/JGLOBAL_OCEANICE_PRODUCTS @JesseMeng-NOAA @ChristopherHill-NOAA
 jobs/JGLOBAL_OFFLINE_ATMOS_ANALYSIS @CoryMartin-NOAA
 jobs/JGLOBAL_PREP_EMISSIONS @bbakernoaa
-jobs/JGLOBAL_PREP_OBS_AERO @CoryMartin-NOAA
 jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA
 jobs/JGLOBAL_*SNOW* @jiaruidong2017
 jobs/JGLOBAL_STAGE_IC @DavidHuber-NOAA @aerorahul
@@ -161,6 +160,7 @@ ush/gaussian_sfcanl.sh @BrianCurtis-NOAA
 ush/getdump.sh @DavidHuber-NOAA @aerorahul 
 ush/getges.sh @DavidHuber-NOAA @aerorahul 
 ush/getgfsnctime @CoryMartin-NOAA
+ush/getioda.sh @nicholasesposito @CoryMartin-NOAA @RussTreadon-NOAA
 ush/getncdimlen @CoryMartin-NOAA
 ush/gfs_bfr2gpk.sh @BoCui-NOAA
 ush/gfs_bufr.sh @BoCui-NOAA
@@ -209,7 +209,6 @@ ush/python/pygfs/task/__init__.py @aerorahul
 ush/python/pygfs/task/aero_analysis.py @DavidNew-NOAA @CoryMartin-NOAA
 ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA
 ush/python/pygfs/task/aero_emissions.py @bbakernoaa
-ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA
 ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA
 ush/python/pygfs/task/analysis_stats.py @CoryMartin-NOAA @DavidNew-NOAA
 ush/python/pygfs/task/archive.py @DavidHuber-NOAA

.gitignore

@@ -169,18 +169,19 @@ ush/fv3gfs_make_orog.sh
 ush/global_chgres.sh
 ush/global_chgres_driver.sh
 ush/global_cycle_driver.sh
+ush/gsi_satbias2ioda_all.sh
 ush/jediinc2fv3.py
 ush/imsfv3_scf2ioda.py
 ush/bufr_snocvr_snomad.py
 ush/atparse.bash
-ush/run_bufr2ioda.py
 ush/bufr2ioda_insitu*
 ush/python/gsincdiag_to_ioda
 ush/python/ufsda
 ush/python/pyiodaconv
 ush/python/soca
 ush/python/gen_bufr2ioda_json.py
 ush/python/gen_bufr2ioda_yaml.py
+ush/python/run_bufr2ioda.py
 ush/spoc
 # ush log file
 ush/fetch-fix-data.log

dev/ci/cases/pr/C96C48_ufsgsi_hybatmDA.yaml

@@ -0,0 +1,31 @@
+experiment:
+  net: gfs
+  mode: cycled
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  idate: 2024022318
+  edate: 2024022406
+  nens: 2
+  interval: 24
+  start: warm
+  yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/ufsgsi_hybatmDA_defaults.ci.yaml
+
+skip_ci_on_hosts:
+  - gaeac5
+  - gaeac6
+  - orion
+  - hercules
+  - awsepicglobalworkflow
+
+workflow:
+  engine: rocoto
+  rocoto:
+    maxtries: 2
+    cyclethrottle: 3
+    taskthrottle: 25
+    verbosity: 2

dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml

@@ -3,3 +3,4 @@ defaults:
 base:
   DO_TEST_MODE: "NO"
   AERO_ANL_RUNS: "gcdas"
+  USE_IODADIR: "YES"

dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -5,6 +5,8 @@ base:
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
   DO_TEST_MODE: "NO"
+  CASE_ANL: "C48"
+  USE_IODADIR: "YES"
 atmanl:
   LAYOUT_X_ATMANL: 4
   LAYOUT_Y_ATMANL: 4
@@ -23,5 +25,7 @@ esfc:
   DONST: "NO"
 nsst:
   NST_MODEL: "1"
+prepatmiodaobs:
+  DO_CONVERT_IODA: "YES"
 sfcanl:
   DONST: "NO"

dev/ci/cases/yamls/ufsgsi_hybatmDA_defaults.ci.yaml

@@ -0,0 +1,32 @@
+defaults:
+  !INC {{ HOMEgfs }}/dev/parm/config/gfs/yaml/defaults.yaml
+base:
+  DOIAU: "NO"
+  DO_JEDIATMVAR: "YES"
+  DO_JEDIATMENS: "NO"
+  DO_TEST_MODE: "NO"
+  CASE_ANL: "C48"
+  CASE_HIST: "C96"
+  DONST: "NO"
+  USE_IODADIR: "YES"
+atmanl:
+  LAYOUT_X_ATMANL: 4
+  LAYOUT_Y_ATMANL: 4
+  OBS_LIST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/atm_obs_list_ufs_hybatmDA.yaml.j2"
+  VAR_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_3dvar_ufs_hybatmDA.yaml.j2"
+  FV3INC_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_3dvar-fv3inc_ufs_hybatmDA.yaml.j2"
+  NUMBER_OUTER_LOOPS: 2
+  NINNER_LOOP1: 2
+  NINNER_LOOP2: 4
+esfc:
+  DONST: "NO"
+nsst:
+  NST_MODEL: "1"
+prep:
+  COPY_BIASCOR_SOURCE: "NO"
+  COPY_BIASCOR_STATIC: "YES"
+  CONVERT_BIASCOR: "YES"
+prepatmiodaobs:
+  DO_CONVERT_IODA: "YES"
+sfcanl:
+  DONST: "NO"

dev/ci/gitlab-ci-hosts.yml

@@ -22,7 +22,7 @@
 
 # Template matrices for case lists
 .hera_cases_matrix: &hera_cases
-  - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmsnowDA", "C96C48_hybatmsoilDA", "C96C48_ufs_hybatmDA", "C96C48mx500_S2SW_cyc_gfs", "C96_atm3DVar", "C96_gcafs_cycled", "C96_gcafs_cycled_noDA", "C96mx100_S2S"]
+  - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmsnowDA", "C96C48_hybatmsoilDA", "C96C48_ufsgsi_hybatmDA", "C96C48_ufs_hybatmDA", "C96C48mx500_S2SW_cyc_gfs", "C96_atm3DVar", "C96_gcafs_cycled", "C96_gcafs_cycled_noDA", "C96mx100_S2S"]
 
 .gaeac6_cases_matrix: &gaeac6_cases
   - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmsnowDA", "C96C48_hybatmsoilDA", "C96C48mx500_S2SW_cyc_gfs", "C96_atm3DVar", "C96_gcafs_cycled", "C96_gcafs_cycled_noDA", "C96mx100_S2S"]
@@ -34,7 +34,7 @@
   - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48mx500_S2SW_cyc_gfs", "C96_atm3DVar", "C96mx100_S2S"]
 
 .ursa_cases_matrix: &ursa_cases
-  - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmsnowDA", "C96C48_hybatmsoilDA", "C96C48_ufs_hybatmDA", "C96C48mx500_S2SW_cyc_gfs", "C96_atm3DVar", "C96mx100_S2S"]
+  - caseName: ["C48_ATM", "C48_S2SW", "C48_S2SWA_gefs", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C96C48_hybatmDA", "C96C48_hybatmsnowDA", "C96C48_hybatmsoilDA", "C96C48_ufsgsi_hybatmDA", "C96C48_ufs_hybatmDA", "C96C48mx500_S2SW_cyc_gfs", "C96_atm3DVar", "C96mx100_S2S"]
 
 # Host: Hera - Standard Cases
 setup_experiments-hera:

dev/jobs/prep.sh

@@ -32,15 +32,32 @@ RUN=${RUN_local} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_OBS:COM_OBS_TMPL \
     COMOUT_OBS:COM_OBS_TMPL \
     COMINobsproc:COM_OBSPROC_TMPL \
-    COMIN_TCVITAL:COM_TCVITAL_TMPL
+    COMINobsforge:COM_OBSFORGE_TMPL \
+    COMIN_TCVITAL:COM_TCVITAL_TMPL \
+    COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
     COMOUT_OBS_PREV:COM_OBS_TMPL \
-    COMINobsproc_PREV:COM_OBSPROC_TMPL
+    COMINobsproc_PREV:COM_OBSPROC_TMPL \
+    COMOUT_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL
 
 mkdir -p "${COMOUT_OBS}"
 
 ###############################################################
+# Copy IODA files to ROTDIR
+if [[ ${USE_IODADIR:-"NO"} == "YES" ]]; then
+    "${HOMEgfs}/ush/getioda.sh" "${PDY}" "${cyc}" "${RUN_local}" "${COMINobsforge}" "${COMOUT_OBS}"
+    status=$?
+    if [[ ${status} -ne 0 ]]; then
+        exit "${status}"
+    fi
+fi
+
+if [[ "${RUN_local}" == "gcdas" ]]; then
+    echo "GCDAS only needs IODA files; exiting prep.sh"
+    exit 0
+fi
+
 # Copy dump files to ROTDIR
 "${HOMEgfs}/ush/getdump.sh" "${PDY}" "${cyc}" "${RUN_local}" "${COMINobsproc}" "${COMOUT_OBS}"
 status=$?
@@ -95,6 +112,28 @@ else
     cpfs "${COMINobsproc}/${RUN_local}.t${cyc}z.syndata.tcvitals.tm00" "${COMOUT_OBS}/"
 fi
 
+
+###############################################################
+# If requested, copy bias correction files from source or stait to analysis directories
+# TODO: remove this when JEDI ATM can cycle bias correction coefficents
+if [[ ${RUN} == "gdas" && ${COPY_BIASCOR_SOURCE:-"NO"} == "YES" ]]; then
+    for file in abias abias_pc abias_air; do
+        if [[ -s "${SOURCE_BIASCOR}/${file}.${GDUMP}.${gPDY}${gcyc}" ]]; then
+            cpreq "${SOURCE_BIASCOR}/${file}.${GDUMP}.${gPDY}${gcyc}" "${COMOUT_ATMOS_ANALYSIS_PREV}/${GDUMP}.t${gcyc}z.${file}"
+            cpreq "${SOURCE_BIASCOR}/${file}.${GDUMP}.${gPDY}${gcyc}" "${COMOUT_ATMOS_ANALYSIS_PREV}/${GDUMP}.t${gcyc}z.${file}.txt"
+        fi
+    done
+fi
+if [[ ${RUN} == "gdas" && ${COPY_BIASCOR_STATIC:-"NO"} == "YES" ]]; then
+    for file in abias abias_pc abias_air; do
+        if [[ -s "${COMOUT_ATMOS_ANALYSIS_PREV}/${GDUMP}.t${gcyc}z.${file}.txt" ]]; then
+            mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
+            cpreq "${COMOUT_ATMOS_ANALYSIS_PREV}/${GDUMP}.t${gcyc}z.${file}.txt" "${COMOUT_ATMOS_ANALYSIS}/${GDUMP}.t${cyc}z.${file}.txt"
+        fi
+    done
+fi
+
+
 ###############################################################
 # Generate prepbufr files from dumps and prior gdas guess
 rm -f "${COMOUT_OBS}/${OPREFIX}prepbufr"
@@ -160,6 +199,23 @@ if [[ ${err} -ne 0 ]]; then
     err_exit "Failed to obtain/create ${files}, ABORT!"
 fi
 
+################################################################################ 
+# If requested, create radiance bias correction files for JEDI
+if [[ ${RUN} == "gdas" && ${CONVERT_BIASCOR:-"NO"} == "YES" ]]; then
+    cd "${DATAROOT}" || true
+    "${HOMEgfs}/ush/gsi_satbias2ioda_all.sh"
+    export err=$?
+    if [[ ${err} -ne 0 ]]; then
+        err_exit "gsi_satbias2ioda failed, ABORT!"
+    fi
+    
+    # Remove temporary working directory
+    cd "${DATAROOT}" || true
+    if [[ "${KEEPDATA}" == "NO" ]]; then
+        rm -rf "${DATA}"
+    fi    
+fi
+
 ################################################################################
 # Exit out cleanly