Synchronize with off-git version used for paper

Author: platipodium

Makefile

@@ -126,20 +126,18 @@ install-nuopc:  schism_nuopc_lib
 	cp libschism_cap.a $(SCHISM_BUILD_DIR)/lib
 	#cp $(SCHISM_NUOPC_MODS) $(DESTDIR)
 	cp $(SCHISM_NUOPC_MODS) $(SCHISM_BUILD_DIR)/include/
-	sed 's#@@SCHISM_BUILD_DIR@@#'$(SCHISM_BUILD_DIR)'#g' ./src/schism/schism_nuopc_cap.mk.in > $(DESTDIR)/schism.mk
-	#sed 's#@@SCHISM_BUILD_DIR@@#'$(SCHISM_BUILD_DIR)'#g' ./src/schism/schism_nuopc_cap.mk.in > $(SCHISM_BUILD_DIR)/include/schism.mk
+	sed 's#@@SCHISM_BUILD_DIR@@#'$(SCHISM_BUILD_DIR)'#g' ./src/schism/schism_cmi_nuopc.mk.in > $(DESTDIR)/schism.mk
+	#sed 's#@@SCHISM_BUILD_DIR@@#'$(SCHISM_BUILD_DIR)'#g' ./src/schism/schism_cmi_nuopc.mk.in > $(SCHISM_BUILD_DIR)/include/schism.mk
 
 ##test: concurrent_esmf_test triple_schism multi_schism schism_pdaf
 test: pdaf
 pdaf: schism_pdaf
 
 # Internal make targets for final linking
-SCHISM_NUOPC_MODS=$(addprefix src/schism/,schism_nuopc_util.mod schism_nuopc_cap.mod)
-SCHISM_NUOPC_OBJS=$(addprefix src/schism/,schism_nuopc_util.o schism_nuopc_cap.o)
-SCHISM_ESMF_MODS=$(addprefix src/schism/,schism_esmf_cap.mod)
-SCHISM_ESMF_OBJS=$(addprefix src/schism/,schism_esmf_cap.o)
-SCHISM_MODS=$(addprefix src/schism/,schism_bmi.mod schism_esmf_util.mod)
-SCHISM_OBJS=$(addprefix src/schism/,schism_bmi.o schism_esmf_util.o)
+SCHISM_NUOPC_MODS=$(addprefix src/schism/,schism_bmi.mod schism_nuopc_util.mod schism_esmf_util.mod schism_cmi_nuopc.mod)
+SCHISM_NUOPC_OBJS=$(addprefix src/schism/,schism_bmi.o schism_esmf_util.o schism_nuopc_util.o schism_cmi_nuopc.o)
+SCHISM_MODS=$(addprefix src/schism/,schism_bmi.mod schism_esmf_util.mod schism_cmi_esmf.mod)
+SCHISM_OBJS=$(addprefix src/schism/,schism_bmi.o schism_esmf_util.o schism_cmi_esmf.o)
 PDAF_OBJS=$(addprefix src/PDAF_bindings/,parser_mpi.o mod_parallel_pdaf.o mod_assimilation.o init_parallel_pdaf.o \
             init_pdaf.o init_pdaf_info.o finalize_pdaf.o init_ens_pdaf.o next_observation_pdaf.o \
             distribute_state_pdaf.o prepoststep_ens.o prepoststep_pdaf.o prepoststep_seek.o init_enkf.o init_seek.o init_seik.o \
@@ -152,14 +150,17 @@ PDAF_OBJS=$(addprefix src/PDAF_bindings/,parser_mpi.o mod_parallel_pdaf.o mod_as
 #	$(F90) $(CPPFLAGS) $^ -o $@ $(LDFLAGS) $(LIBS)
 
 ifdef USE_PDAF
-schism_pdaf: $(PDAF_OBJS) $(SCHISM_OBJS) $(SCHISM_ESMF_OBJS) schism_pdaf.o
+schism_pdaf: $(PDAF_OBJS) $(SCHISM_OBJS) schism_pdaf.o
 	$(F90) $(CPPFLAGS) $^ -o $@ $(LDFLAGS) $(LIBS)
 endif
 
-schism_esmf_lib: $(SCHISM_OBJS)  $(SCHISM_ESMF_OBJS) $(EXPAND_TARGETS)
+#schism_esmf_lib: $(SCHISM_OBJS) $(MODEL_OBJS) $(EXPAND_TARGETS)
+#	$(AR) crs libschism_esmf.a  $(SCHISM_OBJS) $(MODEL_OBJS) .objs/*/*.o
+
+schism_esmf_lib: $(SCHISM_OBJS) $(EXPAND_TARGETS)
 	$(AR) crs libschism_esmf.a  $(SCHISM_OBJS) .objs/*/*.o
 
-schism_nuopc_lib: $(SCHISM_OBJS) $(SCHISM_NUOPC_OBJS) $(EXPAND_TARGETS)
+schism_nuopc_lib: $(SCHISM_NUOPC_OBJS) $(EXPAND_TARGETS)
 	$(AR) crs libschism_cap.a  $(SCHISM_NUOPC_OBJS) .objs/*/*.o
 
 expand_schismlibs:
@@ -184,16 +185,9 @@ $(PDAF_OBJS):
 #endif
 
 ifdef USE_PDAF
-$(SCHISM_ESMF_OBJS): $(PDAF_OBJS) $(SCHISM_OBJS)
+$(SCHISM_NUOPC_OBJS): $(PDAF_OBJS)
 else
-$(SCHISM_ESMF_OBJS): $(SCHISM_OBJS)
-endif
-	make -C src/schism esmf
-
-ifdef USE_PDAF
-$(SCHISM_NUOPC_OBJS): $(PDAF_OBJS) $(SCHISM_OBJS)
-else
-$(SCHISM_NUOPC_OBJS): $(SCHISM_OBJS)
+$(SCHISM_NUOPC_OBJS):
 endif
 	make -C src/schism nuopc
 
@@ -202,7 +196,7 @@ $(SCHISM_OBJS): $(PDAF_OBJS)
 else
 $(SCHISM_OBJS):
 endif
-	make -C src/schism common
+	make -C src/schism esmf
 
 #$(MODEL_OBJS):
 #	make -C src/model esmf

concurrent_esmf_test.F90

@@ -32,7 +32,7 @@
 program main
 
   use esmf
-  use schism_esmf_cap, only: schismSetServices => SetServices
+  use schism_cmi_esmf, only: schismSetServices => SetServices
   use atmosphere_cmi_esmf,  only: atmosSetServices => SetServices
 
   implicit none

multi_schism.F90

@@ -32,7 +32,7 @@
 program main
 
   use esmf
-  use schism_esmf_cap, only: schismSetServices => SetServices
+  use schism_cmi_esmf, only: schismSetServices => SetServices
   use schism_esmf_util, only: clockCreateFrmParam
 
   implicit none

schism_pdaf.F90

@@ -1,7 +1,7 @@
 ! This code is a main driver for a coupled SCHISM and PDAF
 ! program for running multiple schism components concurrently in flexible mode
 ! (i.e. multiple tasks can share same PET list). The coupling model interface is
-! schism_esmf_cap.F90 (and interfaces are defined in schism_bmi.F90)
+! schism_cmi_esmf.F90 (and interfaces are defined in schism_bmi.F90)
 !
 ! @copyright (C) 2018, 2019, 2020-2021 Helmholtz-Zentrum Geesthacht
 ! @author Richard Hofmeister
@@ -36,7 +36,7 @@
 program main
 
   use esmf
-  use schism_esmf_cap, only: schismSetServices => SetServices
+  use schism_cmi_esmf, only: schismSetServices => SetServices
 ! use schism_msgp, only: parallel_abort,myrank
 ! use schism_glbl, only: errmsg,tr_el
 ! USE mod_assimilation, &      ! Variables for assimilation

src/Makefile

@@ -41,8 +41,8 @@ endif
 
 MODELS=atmosphere_cmi_nuopc.o
 
-SCHISM_NUOPC=schism_nuopc_cap.o schism_esmf_util.o schism_nuopc_util.o schism_bmi.o
-SCHISM_ESMF=schism_nuopc_cap.o schism_esmf_util.o schism_bmi.o
+SCHISM_NUOPC=schism_cmi_nuopc.o schism_esmf_util.o schism_nuopc_util.o schism_bmi.o
+SCHISM_ESMF=schism_cmi_nuopc.o schism_esmf_util.o schism_bmi.o
 MODEL_OBJS:=$(addprefix ../model/,$(MODELS))
 SCHISM_NUOPC_OBJS:=$(addprefix ../schism/,$(SCHISM_NUOPC))
 SCHISM_ESMF_OBJS:=$(addprefix ../schism/,$(SCHISM_ESMF))
@@ -61,11 +61,11 @@ default: all
 	$(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $(F90FLAGS) -o $@ 	$<
 
 main_esmf: main_esmf.o driver/toplevel_schism_netcdf.o model/atmosphere_cmi_nuopc.o \
-	$(addprefix schism/,schism_esmf_cap.o schism_esmf_util.o schism_bmi.o)
+	$(addprefix schism/,schism_cmi_esmf.o schism_esmf_util.o schism_bmi.o)
 	$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -I. -o $@ $^ $(ESMF_F90ESMFLINKLIBS) $(F90FLAGS) $(MOSSCO_LIB) $(LDFLAGS) $(LIBS)
 
 main_nuopc: main_nuopc.o driver/driver_schism_atm.o model/atmosphere_cmi_nuopc.o \
-	$(addprefix schism/,schism_nuopc_cap.o schism_esmf_util.o schism_nuopc_util.o schism_bmi.o)
+	$(addprefix schism/,schism_cmi_nuopc.o schism_esmf_util.o schism_nuopc_util.o schism_bmi.o)
 	$(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -I. -o $@ $^ $(ESMF_F90ESMFLINKLIBS) $(F90FLAGS) $(LDFLAGS) $(LIBS)
 
 main_esmf.o: driver/toplevel_schism_atm.o main_esmf.F90 Makefile

src/Makefile.nuopc.bak

@@ -22,7 +22,7 @@ $(error ESMFMKFILE has to be set in environment.)
 endif
 include $(ESMFMKFILE)
 
-include schism/schism_nuopc_cap.mk
+include schism/schism_cmi_nuopc.mk
 DEP_FRONTS:= $(DEP_FRONTS) -DFRONT_SCHISM=$(ESMF_DEP_FRONT)
 DEP_INCS:= $(DEP_INCS) $(addprefix -I, $(ESMF_DEP_INCPATH))
 DEP_CMPL_OBJS:= $(DEP_CMPL_OBJS) $(ESMF_DEP_CMPL_OBJS)

src/PDAF_bindings/Makefile

@@ -72,8 +72,8 @@ LDFLAGS+= -L$(SCHISM_BUILD_DIR)/lib -L$(PDAF_BUILD_DIR) -lpdaf-d
 %.o : %.F90
 	$(ESMF_F90COMPILER) -c $(ESMF_F90COMPILEOPTS) $(ESMF_F90COMPILEPATHS) $(ESMF_F90COMPILEFREECPP) $(ESMF_F90COMPILECPPFLAGS) $(F90FLAGS) $<
 
-#schism_nuopc_cap.o: schism_esmf_util.o schism_bmi.o
-#schism_esmf_cap.o: schism_esmf_util.o schism_bmi.o
+#schism_cmi_nuopc.o: schism_esmf_util.o schism_bmi.o
+#schism_cmi_esmf.o: schism_esmf_util.o schism_bmi.o
 #schism_esmf_util.o: schism_bmi.o
 
 .PHONY:  clean default all
@@ -85,7 +85,7 @@ esmf: parser_mpi.o mod_parallel_pdaf.o mod_assimilation.o init_parallel_pdaf.o i
       collect_state_pdaf.o init_dim_obs_pdaf.o obs_op_pdaf.o init_obs_pdaf.o prodrinva_pdaf.o init_obsvar_pdaf.o assimilate_pdaf.o \
       init_dim_obs_f_pdaf.o init_dim_obs_l_pdaf.o obs_op_f_pdaf.o init_obs_f_pdaf.o init_obs_l_pdaf.o prodrinva_l_pdaf.o init_obsvar_l_pdaf.o \
       init_n_domains_pdaf.o init_dim_l_pdaf.o g2l_state_pdaf.o l2g_state_pdaf.o g2l_obs_pdaf.o 
-nuopc: schism_nuopc_cap.o
+nuopc: schism_cmi_nuopc.o
 
 clean:
 	$(RM) PET*.ESMF_LogFile traceout *.nc *.stdout

src/PDAF_bindings/distribute_state_pdaf.F90

@@ -29,7 +29,8 @@ SUBROUTINE distribute_state_pdaf(dim_p, state_p)
 ! !USES:
   use schism_glbl, only: nea,nsa,npa,nvrt,ntracers,idry_e,we,tr_el, &
     & idry_s,su2,sv2,idry,eta2,tr_nd,uu2,vv2,ww2, &
-    & elnode,i34,rkind,kbe,kbs,isidenode,kbp
+    & elnode,i34,rkind,kbe,kbs,isidenode,kbp, &
+    & tempmin,tempmax,saltmin,saltmax
 ! Check only
   use mod_parallel_pdaf, only: mype_model,task_id,filterpe
   use mod_assimilation, only: offset_field_p
@@ -81,6 +82,12 @@ SUBROUTINE distribute_state_pdaf(dim_p, state_p)
        do k=1,nvrt
          itot=itot+1
          tr_nd(j,k,i)=state_p(itot)
+         ! Add limiter to avoid weird analysis
+         if (j==1) then
+            if(tr_nd(j,k,i)<tempmin.or.tr_nd(j,k,i)>tempmax) tr_nd(j,k,i)=max(tempmin,min(tr_nd(j,k,i),tempmax))
+         elseif (j==2) then
+            if(tr_nd(j,k,i)<saltmin.or.tr_nd(j,k,i)>saltmax) tr_nd(j,k,i)=max(saltmin,min(tr_nd(j,k,i),saltmax))
+         end if
        enddo !k
        if (ifill.eq.1) then
 !      Fill -9999 to see difference of analysis

src/PDAF_bindings/init_dim_obs_f_pdaf.F90

@@ -23,7 +23,7 @@ SUBROUTINE init_dim_obs_f_pdaf(step, dim_obs_f)
 !
 ! !USES:
 ! SCHISM module
-  use schism_glbl, only: nea,ne,ics,dp,elnode,rearth_eq,rearth_pole,xctr,yctr,zctr,eframe,i34,small2,pi,idry_e,rkind,nsteps_from_cold,dt,cumsum_eta
+  use schism_glbl, only: nea,ne,ics,dp,elnode,rearth_eq,rearth_pole,xctr,yctr,zctr,eframe,i34,small2,pi,idry_e,rkind,nsteps_from_cold,dt,cumsum_eta,xnd,ynd
   use schism_msgp, only: parallel_abort
 ! PDAF user define
 ! new28 add in mod_assimilation, add in some schism_interpolation required here
@@ -45,6 +45,7 @@ SUBROUTINE init_dim_obs_f_pdaf(step, dim_obs_f)
   real(rkind), allocatable :: rmsval(:)
   integer nobs,i,l,itmp,ifl,iobs,istat,j,nd,ifiletype
   real(rkind) tmp,xtmp,ytmp,xobsl,yobsl,zcomp,xoblast,yoblast
+  real(rkind) xndmax,xndmin,yndmax,yndmin
   logical fexist
 
 ! !CALLING SEQUENCE:
@@ -126,10 +127,38 @@ SUBROUTINE init_dim_obs_f_pdaf(step, dim_obs_f)
 
 ! Find parent elements in argumented
   iep_obs=0 !flag for no-parent
+  !Find sub domain max/min
+! do i=1,ne
+!    do j=1,i34(i)
+!       if ((i.eq.1).and.(j.eq.1)) then ! initialize
+!          xndmax=xnd(elnode(j,i))
+!          xndmin=xnd(elnode(j,i))
+!          yndmax=ynd(elnode(j,i))
+!          yndmin=ynd(elnode(j,i))
+!       end if
+!       if (xnd(elnode(j,i)).gt.xndmax) xndmax=xnd(elnode(j,i))
+!       if (xnd(elnode(j,i)).lt.xndmin) xndmin=xnd(elnode(j,i))
+!       if (ynd(elnode(j,i)).gt.yndmax) yndmax=ynd(elnode(j,i))
+!       if (ynd(elnode(j,i)).lt.yndmin) yndmin=ynd(elnode(j,i))
+!    end do
+! end do
+! write(*,'(a,4f14.3)') 'Check sub domain!', xndmin, xndmax, yndmin, yndmax
+! !Pre-select to speed up
+! do l=1,nobs
+!     xobsl=xobs(l)
+!     yobsl=yobs(l)
+!     if ((xobsl.gt.xndmin).and.(xobsl.lt.xndmax).and.&
+!        &(yobsl.gt.yndmin).and.(yobsl.lt.yndmax)) then
+!        iep_obs(l)=1
+!        write(*,*) 'iep_obs=1',l,xobsl,yobsl
+!     end if
+! end do
+
   do i=1,ne ! search in resident domain to avoid overlap use of observations
      if(idry_e(i)==1) cycle ! skip dry points
      do l=1,nobs
           if(iep_obs(l)/=0) cycle
+!         if(iep_obs(l)==0) cycle ! skip to speedup searching
 
           if(ics==1) then
              xobsl=xobs(l)
@@ -169,7 +198,9 @@ SUBROUTINE init_dim_obs_f_pdaf(step, dim_obs_f)
                        end if
                     end do
                  end if
-             endif
+!            else  !tmp
+!              iep_obs(l)=0
+             endif !tmp
           else !quad
              call quad_shape(0,0,i,xobsl,yobsl,itmp,arco_obs(l,1:4)) !arco_sta are 4 shape functions
              if(itmp/=0) iep_obs(l)=i
@@ -195,6 +226,9 @@ SUBROUTINE init_dim_obs_f_pdaf(step, dim_obs_f)
                      end if
                   end do
                end if
+!            else !itmp
+!              iep_obs(l)=0
+!            end if !itmp
           endif !i34
           !Check nsteps_from_cold for SSH-A, skip if nsteps_from_cold too small
           if ((obstype(l).eq.'a').or.(obstype(l).eq.'A')) then

src/PDAF_bindings/init_dim_obs_pdaf.F90

@@ -23,7 +23,7 @@ SUBROUTINE init_dim_obs_pdaf(step, dim_obs_p)
 !
 ! !USES:
 ! SCHISM module
-  use schism_glbl, only: nea,ne,ics,dp,elnode,rearth_eq,rearth_pole,xctr,yctr,zctr,eframe,i34,small2,pi,idry_e,rkind,nsteps_from_cold,dt
+  use schism_glbl, only: nea,ne,ics,dp,elnode,rearth_eq,rearth_pole,xctr,yctr,zctr,eframe,i34,small2,pi,idry_e,rkind,nsteps_from_cold,dt,xnd,ynd
   use schism_msgp, only: parallel_abort
 ! PDAF user define
 ! new28 add in mod_assimilation, add in some schism_interpolation required here
@@ -45,6 +45,7 @@ SUBROUTINE init_dim_obs_pdaf(step, dim_obs_p)
   real(rkind), allocatable :: rmsval(:)
   integer nobs,i,l,itmp,ifl,iobs,istat,j,nd
   real(rkind) tmp,xtmp,ytmp,xobsl,yobsl,zcomp
+  real(rkind) xndmax,xndmin,yndmax,yndmin
   logical fexist
 
 ! !CALLING SEQUENCE:
@@ -108,10 +109,34 @@ SUBROUTINE init_dim_obs_pdaf(step, dim_obs_p)
 
 ! Find parent elements in argumented
   iep_obs=0 !flag for no-parent
+  !Find sub domain max/min
+! do i=1,ne
+!    do j=1,i34(i)
+!       if ((i.eq.1).and.(j.eq.1)) then ! initialize
+!          xndmax=xnd(elnode(j,i))
+!          xndmin=xnd(elnode(j,i))
+!          yndmax=ynd(elnode(j,i))
+!          yndmin=ynd(elnode(j,i))
+!       end if
+!       if (xnd(elnode(j,i)).gt.xndmax) xndmax=xnd(elnode(j,i))
+!       if (xnd(elnode(j,i)).lt.xndmin) xndmin=xnd(elnode(j,i))
+!       if (ynd(elnode(j,i)).gt.yndmax) yndmax=ynd(elnode(j,i))
+!       if (ynd(elnode(j,i)).lt.yndmin) yndmin=ynd(elnode(j,i))
+!    end do
+! end do
+  !Pre-select to speed up
+! do l=1,nobs
+!     xobsl=xobs(l)
+!     yobsl=yobs(l)
+!     if ((xobsl.gt.xndmin).and.(xobsl.lt.xndmax).and.&
+!        &(yobsl.gt.yndmin).and.(yobsl.lt.yndmax)) iep_obs(l)=1
+! end do
+     
   do i=1,ne ! search in resident domain to avoid overlap use of observations
      if(idry_e(i)==1) cycle ! skip dry points 
      do l=1,nobs
           if(iep_obs(l)/=0) cycle
+!         if(iep_obs(l)==0) cycle ! skip to speedup searching
 
           if(ics==1) then
              xobsl=xobs(l)