Skip to content
Draft
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 51 additions & 3 deletions mitgcm_code/the_main_loop.F
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,16 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
C \ev

C !USES:
#ifdef ALLOW_AUTODIFF_TAMC
# ifdef ALLOW_IHOP
use arr_mod, only: arr, narr
use bdry_mod, only: bdry
use ssp_mod, only: ssp
use srpos_mod, only: pos
use angle_mod, only: angles
use ihop_mod, only: beam, ray2d
# endif
#endif
IMPLICIT NONE
C == Global variables ==
#include "SIZE.h"
Expand Down Expand Up @@ -443,6 +453,10 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
CALL AUTODIFF_STORE( myThid )
#include "checkpoint_lev4_directives.h"
# ifdef ALLOW_IHOP
!$TAF STORE arr, bdry, narr, pos, ssp = tapelev4, key = ilev_4
!$TAF STORE angles, beam, ray2d = tapelev4, key = ilev_4
# endif
CALL AUTODIFF_RESTORE( myThid )
# ifdef ALLOW_AUTODIFF_WHTAPEIO
CALL AUTODIFF_WHTAPEIO_SYNC( 4 , 1, myThid )
Expand All @@ -462,6 +476,10 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
CALL AUTODIFF_STORE( myThid )
#include "checkpoint_lev3_directives.h"
# ifdef ALLOW_IHOP
!$TAF STORE arr, bdry, narr, pos, ssp = tapelev3, key = ilev_3
!$TAF STORE angles, beam, ray2d = tapelev3, key = ilev_3
# endif
CALL AUTODIFF_RESTORE( myThid )
# ifdef ALLOW_AUTODIFF_WHTAPEIO
CALL AUTODIFF_WHTAPEIO_SYNC( 3 , 1, myThid )
Expand All @@ -480,6 +498,18 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
CALL AUTODIFF_STORE( myThid )
#include "checkpoint_lev2_directives.h"
# ifdef ALLOW_IHOP
CML!$TAF STORE arr, bdry, narr, pos, ssp = tapelev2, key = ilev_2
CML!$TAF STORE angles, beam, ray2d = tapelev2, key = ilev_2
!$TAF STORE arr, narr, ray2d = tapelev2, key = ilev_2
!$TAF STORE angles%dalpha = tapelev2, key = ilev_2
!$TAF STORE bdry%bot, bdry%top = tapelev2, key = ilev_2
!$TAF STORE beam%nsteps = tapelev2, key = ilev_2
!$TAF STORE pos%rz, pos%sz = tapelev2, key = ilev_2
!$TAF STORE ssp%c, ssp%cmat = tapelev2, key = ilev_2
!$TAF STORE ssp%cz, ssp%czmat = tapelev2, key = ilev_2
!$TAF STORE ssp%rho = tapelev2, key = ilev_2
Comment on lines +510 to +517
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be done in a similar way for tapelevel3 and tapelevel4.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use the PASSIVE statement does not work properly

# endif
CALL AUTODIFF_RESTORE( myThid )
# ifdef ALLOW_AUTODIFF_WHTAPEIO
CALL AUTODIFF_WHTAPEIO_SYNC( 2 , 1, myThid )
Expand Down Expand Up @@ -520,7 +550,8 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
c--
# ifdef ALLOW_IHOP
! for ssp_mod>gcmSSP; make local?
!$TAF INIT comlev1_bibj_ij_ihop = COMMON, nchklev_1*nSx*nSy*(2*OLy+sNy)*(2*OLx+sNx)
!$TAF INIT comlev1_ihop = STATIC, nchklev_1
!$TAF INIT comlev1_ihop_nts = STATIC, nchklev_1*nts
# endif /* ALLOW_IHOP */
c--
# ifdef ALLOW_MOM_COMMON
Expand Down Expand Up @@ -698,16 +729,29 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
ENDIF
#endif
#ifdef ALLOW_IHOP
# ifdef ALLOW_AUTODIFF_TAMC
CML!$TAF STORE arr,narr,bdry,pos,ssp = comlev1_ihop, key = ikey_dynamics
!$TAF STORE arr, narr, ray2d = comlev1_ihop, key = ikey_dynamics
!$TAF STORE angles%dalpha = comlev1_ihop, key = ikey_dynamics
!$TAF STORE bdry%bot, bdry%top = comlev1_ihop, key = ikey_dynamics
!$TAF STORE beam%nsteps = comlev1_ihop, key = ikey_dynamics
!$TAF STORE pos%rz, pos%sz = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp%c, ssp%cmat = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp%cz, ssp%czmat = comlev1_ihop, key = ikey_dynamics
C for some reason TAF does not do this right, so we store ssp instead
CML!$TAF STORE ssp%rho = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp = comlev1_ihop, key = ikey_dynamics
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As long as ssp is stored, we don't need the rest

Suggested change
!$TAF STORE ssp%c, ssp%cmat = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp%cz, ssp%czmat = comlev1_ihop, key = ikey_dynamics
C for some reason TAF does not do this right, so we store ssp instead
CML!$TAF STORE ssp%rho = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp = comlev1_ihop, key = ikey_dynamics
CML!$TAF STORE ssp%c, ssp%cmat = comlev1_ihop, key = ikey_dynamics
CML!$TAF STORE ssp%cz, ssp%czmat = comlev1_ihop, key = ikey_dynamics
C for some reason TAF does not do this right, so we store ssp instead
CML!$TAF STORE ssp%rho = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp = comlev1_ihop, key = ikey_dynamics

#endif
IF (useIHOP) THEN
#ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ihop_cost_inloop',myThid)
#endif
c-- Accumulate in-situ acoutsic travel times.
#ifdef ALLOW_AUTODIFF
# ifdef ALLOW_AUTODIFF
C-- Reset the model iteration counter and the model time.
myIter = nIter0 + (iloop-1)
myTime = startTime + deltaTClock*(iloop-1)
#endif
# endif
CALL TIMER_START('IHOP_COST_INLOOP [MAIN_DO_LOOP]', myThid)
CALL IHOP_COST_INLOOP( myTime, myThid )
CALL TIMER_STOP ('IHOP_COST_INLOOP [MAIN_DO_LOOP]', myThid)
Expand Down Expand Up @@ -771,6 +815,10 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
#endif
#ifdef ALLOW_IHOP
IF (useIHOP) THEN
# ifdef ALLOW_AUTODIFF
!$TAF STORE arr, narr = comlev1_ihop, key = ikey_dynamics
!$TAF STORE pos%rz, pos%sz = comlev1_ihop, key = ikey_dynamics
# endif
c-- Accumulate travel times
CALL TIMER_START('IHOP_COST_INLOOP [THE_MAIN_LOOP]', myThid)
CALL IHOP_COST_INLOOP( endtime, myThid )
Expand Down
18 changes: 15 additions & 3 deletions src/ihop.F90
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
#include "IHOP_OPTIONS.h"
#ifdef ALLOW_AUTODIFF
# include "AUTODIFF_OPTIONS.h"
#endif
!BOP
! !ROUTINE: IHOP
! !INTERFACE:
Expand Down Expand Up @@ -59,10 +62,16 @@ SUBROUTINE IHOP_MAIN ( myTime, myIter, myThid )
USE ihop_init_diag, only: initPRTFile, openOutputFiles, resetMemory
USE bdry_mod, only: Bdry, writeBdry
USE ssp_mod, only: setSSP
USE refCoef, only: writeRefCoef
#ifdef ALLOW_AUTODIFF_TAMC
USE ssp_mod, only: SSP
#endif
USE refCoef, only: writeRefCoef
USE beampat, only: writePat
USE ihop_mod, only: Beam

#ifdef ALLOW_AUTODIFF
# include "tamc.h"
#endif
! == Routine Arguments ==
! myThid :: Thread number. Unused by IESCO
! msgBuf :: Used to build messages for printing.
Expand Down Expand Up @@ -91,16 +100,19 @@ SUBROUTINE IHOP_MAIN ( myTime, myIter, myThid )
CALL writeBdry ( myThid )

! write refCoef: OPTIONAL
CALL writeRefCoef( myThid )
CALL writeRefCoef( myThid )

! Source Beam Pattern: OPTIONAL, default is omni source pattern
CALL writePat( myThid )


! set SSP%cmat from gcm SSP: REQUIRED
#ifdef ALLOW_AUTODIFF_TAMC
!$TAF STORE ssp%c, ssp%cz = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp%czmat, ssp%rho = comlev1_ihop, key = ikey_dynamics
#endif
CALL setSSP( myThid )


! open all output files
IF ( IHOP_dumpfreq .GE. 0 ) &
CALL OpenOutputFiles( IHOP_fileroot, myTime, myIter, myThid )
Expand Down
4 changes: 0 additions & 4 deletions src/ihop_cost_inloop.F
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,6 @@ SUBROUTINE IHOP_COST_INLOOP( myTime, myThid )
& .AND.(ihopObs_time(num_file,m,bi,bj).LT.
& (myTime+deltaTclock))) THEN

#ifdef ALLOW_AUTODIFF_TAMC
!$TAF STORE ihop_modval = comlev1_bibj, key=itdkey, byte=isbyte
#endif

ihop_modval=0

CALL ihop_cost_modval(ihop_modval, num_file,
Expand Down
27 changes: 27 additions & 0 deletions src/ihop_driver.F
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
#include "IHOP_OPTIONS.h"
#ifdef ALLOW_AUTODIFF
# include "AUTODIFF_OPTIONS.h"
#endif

CBOP
C !ROUTINE: IHOP_DRIVER
Expand All @@ -11,6 +14,14 @@ SUBROUTINE IHOP_DRIVER( myTime, myIter, myThid )

C !FORTRAN90 USE MODULES: ===============================================
USE ihop, only: ihop_main
#ifdef ALLOW_AUTODIFF_TAMC
USE arr_mod, only: arr
USE bdry_mod, only: bdry
USE ssp_mod, only: ssp
USE srpos_mod, only: pos
use angle_mod, only: angles
use ihop_mod, only: beam, ray2d
#endif

IMPLICIT NONE
C !GLOBAL VARIABLES: ===============================================
Expand All @@ -24,6 +35,9 @@ SUBROUTINE IHOP_DRIVER( myTime, myIter, myThid )
# include "IHOP_SIZE.h"
# include "IHOP.h"
#endif /* ALLOW_IHOP */
#ifdef ALLOW_AUTODIFF_TAMC
# include "tamc.h"
#endif

C !INPUT PARAMETERS: ===================================================
C myTime :: Current time in simulation
Expand All @@ -36,10 +50,23 @@ SUBROUTINE IHOP_DRIVER( myTime, myIter, myThid )

C !LOCAL VARIABLES: ====================================================
INTEGER t
#ifdef ALLOW_AUTODIFF_TAMC
integer tkey
#endif

CEOP

#ifdef ALLOW_IHOP
DO t=1,nts
CML# ifdef ALLOW_AUTODIFF_TAMC
CML tkey = nts*(ikey_dynamics-1) + t
CML!$TAF STORE arr, bdry, ray2d = comlev1_ihop_nts, key = tkey
CML!$TAF STORE angles%dalpha = comlev1_ihop_nts, key = tkey
CML!$TAF STORE bdry%bot, bdry%top = comlev1_ihop_nts, key = tkey
CML!$TAF STORE beam%nsteps = comlev1_ihop_nts, key = tkey
CML!$TAF STORE pos%rz, pos%sz = comlev1_ihop_nts, key = tkey
CML!$TAF STORE ssp%c = comlev1_ihop_nts, key = tkey
CML# endif
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the commented part that leads to the segmentation fault at TAF's end.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This may lead to issues due to ray2d and other derived types being extended into arrays. TAF seems to have a hard time with allocatable arrays within an array of derived types.

One way around this is to create a bunch of temporary arrays to store into tapes.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm, I tried different subsets of the full set, but it always gave the segmentation fault. I store ray2d in other places without (obvious) problems.

IF ( IHOP_iter(t).GE.0 ) THEN
IF ( IHOP_iter(t).EQ.myIter ) THEN
CALL IHOP_MAIN( myTime, myIter, myThid )
Expand Down
20 changes: 13 additions & 7 deletions src/splinec_mod.F90
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,9 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

L = N - 1

!$TAF init cspline1 = static, 4*ndim*L
!$TAF init cspline1 = static, 1
!$TAF init cspline1_ndim = static, NDIM
!ML!$TAF init cspline1 = static, 4*NDIM*L ! not sure if this correct

DO M = 2,N
C(3,M) = TAU(M) - TAU(M-1)
Expand All @@ -88,7 +90,7 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

! * BEGINNING BOUNDARY CONDITION SECTION *

!$TAF store C = cspline1
!$TAF INCOMPLETE C
IF (IBCBEG==0) THEN ! IBCBEG = 0
!$TAF store C = cspline1
IF (N.GT.2) THEN ! N > 2
Expand Down Expand Up @@ -120,18 +122,21 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

! * RUNNING CALCULATIONS TO N-1 - LOOP IS NOT EXECUTED IF N = 2 *

DO M = 2,L
!$TAF store C = cspline1
DO M = 2,L
!$TAF store C = cspline1_ndim, key = M
G = -C(3,M+1) / C(4,M-1)
C(2,M) = G*C(2,M-1) + 3.0*(C(3,M)*C(4,M+1) + C(3,M+1)*C(4,M))
C(4,M) = G*C(3,M-1) + 2.0*(C(3,M) + C(3,M+1))
END DO

! * ENDING BOUNDARY CONDITION SECTION *

!$TAF INCOMPLETE C
IF (IBCEND /= 1) THEN
!$TAF store C = cspline1
IF (IBCEND==0) THEN
!$TAF INCOMPLETE C
IF (N==2 .AND. IBCBEG==0) THEN
C(2,N) = C(4,N)
ELSE IF ((N==3 .AND. IBCBEG==0) .OR. N==2) THEN
Expand All @@ -154,17 +159,18 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)
G = -1.0 / C(4,N-1)
ELSE
! do nothing
C(2,N) = C(2,N)
C(4,N) = C(4,N)
C(2,N) = C(2,N)
C(4,N) = C(4,N)
G = G
END IF
ENDIF

IF ( IBCBEG.GT.0 .OR. N.GT.2) THEN
!$TAF store C = cspline1
C(4,N) = G*C(3,N-1) + C(4,N)
C(2,N) = (G*C(2,N-1) + C(2,N)) / C(4,N)
END IF
END IF
!$TAF store C = cspline1

! * RUN THE ENDING BOUNDARY EFFECT BACK THROUGH THE EQUATIONS *

Expand Down Expand Up @@ -194,7 +200,7 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

C(4,N) = (0.0,0.0)
DO I = 1,L ! INTEGRATE OVER THE INTERVAL
!$TAF store C = cspline1
!$TAF store C = cspline1_ndim, key = I
DTAU = TAU(I+1) - TAU(I)
C(4,N) = C(4,N) + DTAU*(C(1,I) + DTAU*(C(2,I)/2.0 + &
DTAU*(C(3,I)/6.0 + DTAU*C(4,I)/24.0)))
Expand Down
Loading