From 020d7e4038af9a5dbcbf9c4bc38771ca76be1c97 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Fri, 4 Dec 2020 12:03:09 -0800 Subject: [PATCH 1/5] Forward and reverse models compile and run --- MAPL_cfio/ESMF_CFIOMod.F90 | 28 +++- base/MAPL_CapGridComp.F90 | 284 +++++++++++++++++++++++++++++++++- base/MAPL_CapOptions.F90 | 2 +- base/MAPL_Generic.F90 | 92 ++++++++--- base/MAPL_GenericCplComp.F90 | 57 ++++++- base/MAPL_HistoryGridComp.F90 | 277 +++++++++++++++++++++++++++++---- base/MAPL_IO.F90 | 41 ++++- 7 files changed, 715 insertions(+), 66 deletions(-) diff --git a/MAPL_cfio/ESMF_CFIOMod.F90 b/MAPL_cfio/ESMF_CFIOMod.F90 index ffa11588de6..91c4139366a 100644 --- a/MAPL_cfio/ESMF_CFIOMod.F90 +++ b/MAPL_cfio/ESMF_CFIOMod.F90 @@ -790,7 +790,6 @@ end subroutine ESMF_CFIOVarWrite1D_ ! !INTERFACE: subroutine ESMF_CFIOFileOpen (cfio, fmode, rc, expid, cyclic) - ! ! !ARGUMENTS: ! @@ -839,7 +838,11 @@ subroutine ESMF_CFIOFileOpen (cfio, fmode, rc, expid, cyclic) character(len=16) :: format logical :: ex character(len=MLEN) :: fileName - logical :: myCyclic + logical :: myCyclic + logical :: exists, found, open + integer :: LUN, i + + INTEGER, PARAMETER :: iTop = 199 ! Maximum LUN limit if (present(expid)) call ESMF_CFIOSet(cfio, expid = expid) @@ -855,9 +858,24 @@ subroutine ESMF_CFIOFileOpen (cfio, fmode, rc, expid, cyclic) print *, trim(fileName), "doesn't exist" return end if - open(11, file=fileName) - read(11, '(a)') dset - close(11) + !====================================================================== + ! Find an available logical unit + !====================================================================== + found = .FALSE. + i = 11 + + DO WHILE ( .NOT. found .AND. i <= iTop ) + INQUIRE( UNIT=i, EXIST=exists, OPENED=open ) + IF ( exists .AND. .NOT. open ) THEN + found = .TRUE. + lun = i + ENDIF + i = i + 1 + ENDDO + + open(LUN, file=fileName) + read(LUN, '(a)') dset + close(LUN) format = 'SDF' if (index(dset,'DSET') .ge. 1 .or. index(dset,'dset') .ge. 1 & .or. index(dset,'Dset') .ge. 1 ) then diff --git a/base/MAPL_CapGridComp.F90 b/base/MAPL_CapGridComp.F90 index 6e87f74234f..644991f1524 100644 --- a/base/MAPL_CapGridComp.F90 +++ b/base/MAPL_CapGridComp.F90 @@ -60,6 +60,9 @@ module MAPL_CapGridCompMod procedure :: initialize_history procedure :: run procedure :: step +#ifdef ADJOINT + procedure :: step_reverse +#endif procedure :: finalize procedure :: get_model_duration procedure :: get_am_i_root @@ -169,6 +172,7 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc) character(len=ESMF_MAXSTR ) :: DYCORE character(len=ESMF_MAXPATHLEN) :: user_dirpath,tempString logical :: tend,foundPath + integer :: reverseTime type (MAPL_MetaComp), pointer :: maplobj @@ -467,6 +471,16 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc) call ESMF_ConfigGetAttribute(cap%cf_root, value=ReplayMode, Label="REPLAY_MODE:", default="NoReplay", rc=status) _VERIFY(STATUS) + ! pass REVERSE_TIME resource to history and root config + call MAPL_GetResource(MAPLOBJ, reverseTime, "REVERSE_TIME:", default = 0, rc = status) + _VERIFY(status) + + call MAPL_ConfigSetAttribute(cap%cf_root, value=reverseTime, Label="REVERSE_TIME:", rc=status) + _VERIFY(STATUS) + + call MAPL_ConfigSetAttribute(cap%cf_hist, value=reverseTime, Label="REVERSE_TIME:", rc=status) + _VERIFY(STATUS) + ! Register the children with MAPL !-------------------------------- @@ -981,14 +995,106 @@ subroutine run_MAPL_GridComp(gc, rc) integer :: n, status logical :: done + integer :: reverse_time type(MAPL_CapGridComp), pointer :: cap type (MAPL_MetaComp), pointer :: MAPLOBJ procedure(), pointer :: root_set_services + type(ESMF_Time) :: stopTime, currTime + + + ! instantiate Alarm lists + type(ESMF_Alarm) :: alarm(200), hist_alarm(400) + + ! local variables for Get methods + integer :: ringingAlarmCount ! at any time step (0 to NUMALARMS) + integer :: alarmCount, hist_alarmCount + + ! name, loop counter, result code + character (len=ESMF_MAXSTR) :: name + integer :: i, result cap => get_CapGridComp_from_gc(gc) call MAPL_GetObjectFromGC(gc, maplobj, rc=status) _VERIFY(status) + ! Time Loop starts by checking for Segment Ending Time + !----------------------------------------------------- + ! Check if user wants to reverse time + call MAPL_Set(MAPLOBJ, name = cap%name, cf = cap%config, rc = status) + _VERIFY(status) + call MAPL_GetResource(MAPLOBJ, reverse_time, label='REVERSE_TIME:', & + default=0, rc = status) + _VERIFY(STATUS) + if ( reverse_time == 1 ) then + + call ESMF_ClockGetAlarmList(cap%clock, ESMF_ALARMLIST_ALL, & + alarmList=alarm, alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + + call ESMF_ClockGetAlarmList(cap%clock_hist, ESMF_ALARMLIST_ALL, & + alarmList=hist_alarm, alarmCount=hist_alarmCount, rc = status ) + _VERIFY(STATUS) + + if (MAPL_Am_I_Root()) THEN + + WRITE(*,1003) 'clock', alarmCount + + WRITE(*,1003) 'clock_hist', hist_alarmCount + + WRITE(*,1001) cap%nsteps + endif +1001 FORMAT(' MAPL_CapGC running for ', i3, ' timesteps') +1003 FORMAT(3x, a10, ' has ', i3, ' alarms') + ! Need to run time loop forwards first so alarms are + ! called properly on reverse time run + ! Time Loop starts by checking for Segment Ending Time + !----------------------------------------------------- + if (.true.) then + FAKE_TIME_LOOP: do n = 1, cap%nsteps + + if (MAPL_Am_I_Root()) & + WRITE(*,1002) n, cap%nsteps +1002 FORMAT(' MAPL_CapGC running step ', i3, ' of ', i3) + + if (.not.cap%lperp) then + done = ESMF_ClockIsDone(cap%clock_hist, rc = status) + _VERIFY(status) + if (done .and. MAPL_Am_I_Root()) & + WRITE(*,*) ' MAPL_CapGC: No perpetual clock and history is done. Exiting loop' + if (done) exit + endif + + + ! Advance the Clock before running History and Record + ! --------------------------------------------------- + + call ESMF_ClockAdvance(cap%clock, ringingAlarmCount=ringingAlarmCount, rc = status) + _VERIFY(STATUS) + + call ESMF_ClockAdvance(cap%clock_hist, ringingAlarmCount=ringingAlarmCount, rc = status) + _VERIFY(STATUS) + + enddo FAKE_TIME_LOOP ! end of time loop + ! Update the cap_restart_time variable to avoid crash on last timestep + call ESMF_ClockGetNextTime(cap%clock,nextTime=cap%cap_restart_time,rc=status) + _VERIFY(status) + endif + if (MAPL_Am_I_Root()) & + WRITE(*,*) ' MAPL_CapGC finished time loop, reversing clocks' + + if (MAPL_Am_I_Root()) & + WRITE(*,*) ' MAPL_CapGC reversing clock' + ! Reverse the direction of clock. Now the calls to ESMF_ClockAdvance + ! would tick the clock backwards + call ESMF_ClockSet ( cap%clock, direction=ESMF_DIRECTION_REVERSE, & + advanceCount=0, rc=status ) + _VERIFY(STATUS) + + ! We also have to do this for the History clock + call ESMF_ClockSet ( cap%clock_hist, direction=ESMF_DIRECTION_REVERSE, & + advancecount=0, rc=status ) + _VERIFY(STATUS) + endif if (.not. cap%printspec > 0) then @@ -998,18 +1104,40 @@ subroutine run_MAPL_GridComp(gc, rc) _VERIFY(status) cap%loop_start_timer = MPI_WTime(status) cap%started_loop_timer = .true. - TIME_LOOP: do n = 1, cap%nsteps + TIME_LOOP: do n = 1, cap%nsteps+1 call MAPL_MemUtilsWrite(cap%vm, 'MAPL_Cap:TimeLoop', rc = status) _VERIFY(status) if (.not.cap%lperp) then - done = ESMF_ClockIsStopTime(cap%clock_hist, rc = status) + if ( reverse_time == 0 ) then + done = ESMF_ClockIsStopTime(cap%clock_hist, rc = status) + else + done = ESMF_ClockIsDone(cap%clock_hist, rc = status) + endif _VERIFY(status) + if (MAPL_Am_I_Root() .and. done) THEN + call ESMF_ClockPrint(cap%clock_hist, options='currTime string', rc = status) + _VERIFY(status) + call ESMF_ClockPrint(cap%clock_hist, options='stopTime string', rc = status) + _VERIFY(status) + call ESMF_ClockPrint(cap%clock_hist, options='direction', rc = status) + _VERIFY(status) + endif if (done) exit endif +#ifdef ADJOINT + if (.not. reverse_time) THEN +#endif call cap%step(status) +#ifdef ADJOINT + ELSE + if (MAPL_Am_I_Root()) & + print *, 'Calling step_reverse' + call cap%step_reverse(n .eq. 1, status) + ENDIF +#endif _VERIFY(status) ! Reset loop average timer to get a better @@ -1139,10 +1267,13 @@ subroutine step(this, rc) n=delt64/real(this%heartbeat_dt,kind=real64) ! GridCompRun Timer [Inst] RUN_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_RUN_TIMER-START_RUN_TIMER) + WRITE(*,*) 'RUN_THROUGHPUT = ', RUN_THROUGHPUT ! Time loop throughput [Inst] INST_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER-START_TIMER) + WRITE(*,*) 'INST_THROUGHPUT = ', INST_THROUGHPUT ! Time loop throughput [Avg] LOOP_THROUGHPUT = REAL(n*this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER- this%loop_start_timer) + WRITE(*,*) 'LOOP_THROUGHPUT = ', LOOP_THROUGHPUT ! Estimate time remaining (seconds) TIME_REMAINING = REAL((this%nsteps-n)*this%HEARTBEAT_DT,kind=REAL64)/LOOP_THROUGHPUT HRS_R = FLOOR(TIME_REMAINING/3600.0) @@ -1167,7 +1298,152 @@ subroutine step(this, rc) _RETURN(ESMF_SUCCESS) end subroutine step +#ifdef ADJOINT + subroutine step_reverse(this, first, rc) + class(MAPL_CapGridComp), intent(inout) :: this + logical, intent(in) :: first + integer, intent(out) :: rc + integer :: AGCM_YY, AGCM_MM, AGCM_DD, AGCM_H, AGCM_M, AGCM_S + integer :: status +! For Throughout/execution estimates + integer :: HRS_R, MIN_R, SEC_R + real(kind=REAL64) :: START_RUN_TIMER,END_RUN_TIMER,START_TIMER,END_TIMER + real(kind=REAL64) :: TIME_REMAINING + real(kind=REAL64) :: LOOP_THROUGHPUT=0.0_REAL64 + real(kind=REAL64) :: INST_THROUGHPUT=0.0_REAL64 + real(kind=REAL64) :: RUN_THROUGHPUT=0.0_REAL64 + real :: mem_total, mem_commit, mem_committed_percent + real :: mem_used, mem_used_percent + + + type(ESMF_Time) :: currTime + type(ESMF_TimeInterval) :: delt + real(kind=REAL64) :: delt64 + integer :: n + + call ESMF_GridCompGet(this%gc, vm = this%vm) + + if (.not.this%started_loop_timer) then + this%loop_start_timer = MPI_WTime(status) + this%started_loop_timer=.true. + end if + start_timer = MPI_Wtime(status) + ! Run the ExtData Component + ! -------------------------- + + call ESMF_GridCompRun(this%gcs(this%extdata_id), importState = this%child_imports(this%extdata_id), & + exportState = this%child_exports(this%extdata_id), & + clock = this%clock, userrc = status) + _VERIFY(status) + + ! Call Record for intermediate checkpoint (if desired) + ! Note that we are not doing a Record for History. + ! ------------------------------------------------------ + ! don't output reverse checkpoints + ! call ESMF_GridCompWriteRestart(this%gcs(this%root_id), importstate = this%child_imports(this%root_id), & + ! exportstate = this%child_exports(this%root_id), & + ! clock = this%clock_hist, userrc = status) + ! _VERIFY(status) + ! Advance the Clock before running History and Record + ! --------------------------------------------------- + + if (.not. first) THEN + call ESMF_VMBarrier(this%vm,rc=status) + _VERIFY(status) + call ESMF_ClockAdvance(this%clock, rc = status) + _VERIFY(STATUS) + call ESMF_ClockAdvance(this%clock_hist, rc = status) + _VERIFY(STATUS) + end if + + ! Update Perpetual Clock + ! ---------------------- + + if (this%lperp) then + call Perpetual_Clock(this%clock, this%clock_hist, this%perpetual_year, this%perpetual_month, this%perpetual_day, status) + _VERIFY(status) + end if + + call ESMF_ClockGet(this%clock, CurrTime = currTime, rc = status) + _VERIFY(status) + call ESMF_TimeGet(CurrTime, YY = AGCM_YY, & + MM = AGCM_MM, & + DD = AGCM_DD, & + H = AGCM_H , & + M = AGCM_M , & + S = AGCM_S, rc=status) + _VERIFY(status) + delt=this%cap_restart_time-currTime + + ! Run the Gridded Component + ! -------------------------- + start_run_timer = MPI_WTime(status) + call ESMF_GridCompRun(this%gcs(this%root_id), importstate = this%child_imports(this%root_id), & + exportstate = this%child_exports(this%root_id), & + clock = this%clock, userrc = status) + _VERIFY(status) + call ESMF_VMBarrier(this%vm,rc=status) + _VERIFY(status) + end_run_timer = MPI_WTime(status) + + ! Synchronize for Next TimeStep + ! ----------------------------- + + call ESMF_VMBarrier(this%vm, rc = status) + _VERIFY(STATUS) + + ! Call History Run for Output + ! --------------------------- + + call ESMF_GridCompRun(this%gcs(this%history_id), importstate=this%child_imports(this%history_id), & + exportstate = this%child_exports(this%history_id), & + clock = this%clock_hist, userrc = status) + _VERIFY(status) + + ! Estimate throughput times + ! --------------------------- + + ! Call system clock to estimate throughput simulated Days/Day + call ESMF_VMBarrier( this%vm, RC=STATUS ) + _VERIFY(STATUS) + END_TIMER = MPI_Wtime(status) + call ESMF_TimeIntervalGet(delt,s_r8=delt64,rc=status) + _VERIFY(status) + n=delt64/real(this%heartbeat_dt,kind=real64) + ! GridCompRun Timer [Inst] + RUN_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_RUN_TIMER-START_RUN_TIMER) + WRITE(*,*) 'RUN_THROUGHPUT = ', RUN_THROUGHPUT + ! Time loop throughput [Inst] + INST_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER-START_TIMER) + WRITE(*,*) 'INST_THROUGHPUT = ', INST_THROUGHPUT + ! Time loop throughput [Avg] + LOOP_THROUGHPUT = REAL(n*this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER- this%loop_start_timer) + WRITE(*,*) 'LOOP_THROUGHPUT = ', LOOP_THROUGHPUT + ! Estimate time remaining (seconds) + TIME_REMAINING = REAL((this%nsteps-n)*this%HEARTBEAT_DT,kind=REAL64)/LOOP_THROUGHPUT + HRS_R = FLOOR(TIME_REMAINING/3600.0) + MIN_R = FLOOR(TIME_REMAINING/60.0 - 60.0*HRS_R) + SEC_R = FLOOR(TIME_REMAINING - 3600.0*HRS_R - 60.0*MIN_R) + ! Reset Inst timer + START_TIMER=END_TIMER + ! Get percent of used memory + call MAPL_MemUsed ( mem_total, mem_used, mem_used_percent, RC=STATUS ) + _VERIFY(STATUS) + ! Get percent of committed memory + call MAPL_MemCommited ( mem_total, mem_commit, mem_committed_percent, RC=STATUS ) + _VERIFY(STATUS) + + if( mapl_am_I_Root(this%vm) ) write(6,1000) AGCM_YY,AGCM_MM,AGCM_DD,AGCM_H,AGCM_M,AGCM_S,& + LOOP_THROUGHPUT,INST_THROUGHPUT,RUN_THROUGHPUT,HRS_R,MIN_R,SEC_R,& + mem_committed_percent,mem_used_percent + 1000 format(1x,'AGCM Date: ',i4.4,'/',i2.2,'/',i2.2,2x,'Time: ',i2.2,':',i2.2,':',i2.2, & + 2x,'Throughput(days/day)[Avg Tot Run]: ',f6.1,1x,f6.1,1x,f6.1,2x,'TimeRemaining(Est) ',i3.3,':'i2.2,':',i2.2,2x, & + f5.1,'% : ',f5.1,'% Mem Comm:Used') + + _RETURN(ESMF_SUCCESS) + end subroutine step_reverse +#endif ! !IROUTINE: MAPL_ClockInit -- Sets the clock ! !INTERFACE: @@ -1345,7 +1621,7 @@ subroutine MAPL_ClockInit ( MAPLOBJ, Clock, nsteps, rc) _ASSERT(NUM_DT>=0, 'NUM_DT should be >= 0.') _ASSERT(DEN_DT> 0, 'DEN_DT should be > 0.') _ASSERT(NUM_DT=0, 'HEARTBEAT_DT should be >= 0.') + !_ASSERT(HEARTBEAT_DT>=0, 'HEARTBEAT_DT should be >= 0.') ! initialize calendar to be Gregorian type ! ---------------------------------------- @@ -1443,6 +1719,8 @@ subroutine MAPL_ClockInit ( MAPLOBJ, Clock, nsteps, rc) rc = STATUS ) _VERIFY(STATUS) + if (endTime < startTime) duration = duration * -1 + stopTime = currTime + duration ! initialize model time step diff --git a/base/MAPL_CapOptions.F90 b/base/MAPL_CapOptions.F90 index 51eca157e2e..46ec07dae2e 100644 --- a/base/MAPL_CapOptions.F90 +++ b/base/MAPL_CapOptions.F90 @@ -16,7 +16,7 @@ module MAPL_CapOptionsMod logical :: use_comm_world = .true. character(:), allocatable :: egress_file character(:), allocatable :: cap_rc_file - type (ESMF_LogKind_Flag) :: esmf_logging_mode = ESMF_LOGKIND_NONE + type (ESMF_LogKind_Flag) :: esmf_logging_mode = ESMF_LOGKIND_SINGLE integer :: npes_model = -1 ! only one of the next two options can have nonzero values integer, allocatable :: npes_input_server(:) diff --git a/base/MAPL_Generic.F90 b/base/MAPL_Generic.F90 index 7a664d91056..134cb16c491 100644 --- a/base/MAPL_Generic.F90 +++ b/base/MAPL_Generic.F90 @@ -166,9 +166,9 @@ module MAPL_GenericMod public MAPL_TerminateImport ! MAPL_Util - !public MAPL_GenericStateClockOn - !public MAPL_GenericStateClockOff - !public MAPL_GenericStateClockAdd + public MAPL_GenericStateClockOn + public MAPL_GenericStateClockOff + public MAPL_GenericStateClockAdd public MAPL_TimerOn public MAPL_TimerOff public MAPL_TimerAdd @@ -1279,17 +1279,30 @@ recursive subroutine MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, RC ) H = HH, M = MM, S = SS, rc = STATUS ) _VERIFY(STATUS) - if (ringTime > currTime) then - ringTime = ringTime - (INT((ringTime - currTime)/TIMEINT)+1)*TIMEINT - end if + if (timeint == ESMF_TimeIntervalAbsValue(timeint)) then + if (ringTime > currTime) then + ringTime = ringTime - (INT((ringTime - currTime)/TIMEINT)+1)*TIMEINT + end if + else + if (ringTime < currTime) then + ringTime = ringTime - (INT((ringTime - currTime)/TIMEINT)+1)*TIMEINT + end if + endif ringTime = ringTime-TSTEP ! we back off current time with clock's dt since ! we advance the clock AFTER run method -! make sure that ringTime is not in the past - do while (ringTime < currTime) - ringTime = ringTime + TIMEINT - end do + if (timeint == ESMF_TimeIntervalAbsValue(timeint)) then + ! make sure that ringTime is not in the past + do while (ringTime < currTime) + ringTime = ringTime + TIMEINT + end do + else + ! make sure that ringTime is not in the past + do while (ringTime > currTime) + ringTime = ringTime + TIMEINT + end do + endif STATE%ALARM(0) = ESMF_AlarmCreate(CLOCK = CLOCK, & name = trim(COMP_NAME) // "_Alarm" , & @@ -2128,8 +2141,13 @@ recursive subroutine MAPL_GenericFinalize ( GC, IMPORT, EXPORT, CLOCK, RC ) LABEL="INTERNAL_HEADER:", & RC=STATUS) _VERIFY(STATUS) - call MAPL_ESMFStateWriteToFile(STATE%INTERNAL,CLOCK,FILENAME, & - FILETYPE, STATE, hdr/=0, oClients = o_Clients, RC=STATUS) + IF (FILENAME(1:1) == '-' .or. FILENAME(1:1) == '+') THEN + call MAPL_ESMFStateWriteToFile(STATE%INTERNAL,CLOCK,FILENAME(2:), & + FILETYPE, STATE, hdr/=0, oClients = o_Clients, RC=STATUS) + else + call MAPL_ESMFStateWriteToFile(STATE%INTERNAL,CLOCK,FILENAME, & + FILETYPE, STATE, hdr/=0, oClients = o_Clients, RC=STATUS) + endif _VERIFY(STATUS) endif @@ -2152,8 +2170,13 @@ recursive subroutine MAPL_GenericFinalize ( GC, IMPORT, EXPORT, CLOCK, RC ) _ASSERT(.false.,'needs informative message') endif #endif - call MAPL_ESMFStateWriteToFile(IMPORT,CLOCK,FILENAME, & - FILETYPE, STATE, .FALSE., oClients = o_Clients, RC=STATUS) + IF (FILENAME(1:1) == '-' .or. FILENAME(1:1) == '+') THEN + call MAPL_ESMFStateWriteToFile(IMPORT,CLOCK,FILENAME(2:), & + FILETYPE, STATE, .FALSE., oClients = o_Clients, RC=STATUS) + else + call MAPL_ESMFStateWriteToFile(IMPORT,CLOCK,FILENAME, & + FILETYPE, STATE, .FALSE., oClients = o_Clients, RC=STATUS) + endif _VERIFY(STATUS) endif end if @@ -2443,10 +2466,19 @@ subroutine MAPL_StateRecord( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GetResource( STATE, FILETYPE, LABEL="DEFAULT_CHECKPOINT_TYPE:", default='pnc4', RC=STATUS ) _VERIFY(STATUS) end if - call MAPL_ESMFStateWriteToFile(IMPORT, CLOCK, & - STATE%RECORD%IMP_FNAME, & - FILETYPE, STATE, .FALSE., oClients = o_Clients, & - RC=STATUS) + IF (STATE%RECORD%IMP_FNAME(1:1) == '-' .or. STATE%RECORD%IMP_FNAME == '+') THEN + call MAPL_ESMFStateWriteToFile(IMPORT, CLOCK, & + STATE%RECORD%IMP_FNAME(2:), & + FILETYPE, STATE, .FALSE., & + oClients = o_Clients, & + RC=STATUS) + else + call MAPL_ESMFStateWriteToFile(IMPORT, CLOCK, & + STATE%RECORD%IMP_FNAME, & + FILETYPE, STATE, .FALSE., & + oClients = o_Clients, & + RC=STATUS) + endif _VERIFY(STATUS) end if @@ -2458,10 +2490,19 @@ subroutine MAPL_StateRecord( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GetResource( STATE, FILETYPE, LABEL="DEFAULT_CHECKPOINT_TYPE:", default='pnc4', RC=STATUS ) _VERIFY(STATUS) end if - call MAPL_ESMFStateWriteToFile(STATE%INTERNAL, CLOCK, & - STATE%RECORD%INT_FNAME, & - FILETYPE, STATE, hdr/=0, oClients = o_Clients, & - RC=STATUS) + if (STATE%RECORD%INT_FNAME(1:1) == '-' .or. STATE%RECORD%INT_FNAME(1:1) == '+') THEN + call MAPL_ESMFStateWriteToFile(STATE%INTERNAL, CLOCK, & + STATE%RECORD%INT_FNAME(2:), & + FILETYPE, STATE, hdr/=0, & + oClients = o_Clients, & + RC=STATUS) + else + call MAPL_ESMFStateWriteToFile(STATE%INTERNAL, CLOCK, & + STATE%RECORD%INT_FNAME, & + FILETYPE, STATE, hdr/=0, & + oClients = o_Clients, & + RC=STATUS) + endif _VERIFY(STATUS) end if @@ -2669,7 +2710,12 @@ subroutine MAPL_StateRefresh( GC, IMPORT, EXPORT, CLOCK, RC ) STATE%RECORD%INT_FNAME, & STATE, hdr/=0, RC=STATUS) _VERIFY(STATUS) - UNIT = GETFILE(STATE%RECORD%INT_FNAME, RC=STATUS) + IF (STATE%RECORD%INT_FNAME(1:1) .eq. '-' .or. & + STATE%RECORD%INT_FNAME(1:1) .eq. '+') THEN + UNIT = GETFILE(STATE%RECORD%INT_FNAME(2:), RC=STATUS) + else + UNIT = GETFILE(STATE%RECORD%INT_FNAME, RC=STATUS) + endif _VERIFY(STATUS) call MAPL_DestroyFile(unit = UNIT, rc=STATUS) _VERIFY(STATUS) diff --git a/base/MAPL_GenericCplComp.F90 b/base/MAPL_GenericCplComp.F90 index 2485c9ca44f..63d59d09c22 100644 --- a/base/MAPL_GenericCplComp.F90 +++ b/base/MAPL_GenericCplComp.F90 @@ -29,6 +29,7 @@ module MAPL_GenericCplCompMod use MAPL_SunMod use MAPL_VarSpecMod use MAPL_ExceptionHandling + use MAPL_CommsMod, only: MAPL_AM_I_ROOT implicit none private @@ -281,6 +282,11 @@ subroutine Initialize(CC, SRC, DST, CLOCK, RC) type(ESMF_Field) :: field integer :: cplfunc logical :: isPresent +! debug nonsense + logical, parameter :: clock_debug = .false. + character(len=ESMF_MAXSTR) :: tmpstring + integer :: year,month,day,hour,minute,second + ! Begin... @@ -389,6 +395,25 @@ subroutine Initialize(CC, SRC, DST, CLOCK, RC) sticky = .false., & rc=STATUS ) _VERIFY(STATUS) + + if (clock_debug .and. MAPL_AM_I_ROOT()) THEN + call ESMF_TimeGet ( rTime, timeString=tmpstring, rc=status ) ; _VERIFY(STATUS) + + read(tmpstring( 1: 4),'(i4.4)') year + read(tmpstring( 6: 7),'(i2.2)') month + read(tmpstring( 9:10),'(i2.2)') day + read(tmpstring(12:13),'(i2.2)') hour + read(tmpstring(15:16),'(i2.2)') minute +1010 format(1X,"RingTime: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " name:", a18) + write(6,1010) & + year, month, day, hour, minute, trim(COMP_NAME)//'_'//trim(NAME) + + call ESMF_TimeIntervalGet ( tcpl, h=hour, m=minute, s=second, rc=status ) ; _VERIFY(STATUS) +1011 format(1X,"RingInterval: ", i3, " hours, ", i3, " minutes, ", i4, " seconds. name:", a18) + write(6,1011) & + hour, minute, second, trim(COMP_NAME)//'_'//trim(NAME) + write(6,*) ' State interval: ', STATE%COUPLE_INTERVAL(J) + endif if(rTime == currTime) then call ESMF_AlarmRingerOn(STATE%TIME_TO_COUPLE(J), rc=status); _VERIFY(STATUS) @@ -401,13 +426,20 @@ subroutine Initialize(CC, SRC, DST, CLOCK, RC) calendar=cal, RC=STATUS) _VERIFY(STATUS) - if (TCLR < TS) TCLR = TS + if (ESMF_TimeIntervalAbsValue(TCLR) < ESMF_TimeIntervalAbsValue(TS)) & + TCLR = TS rTime = TM0 + TOFF - TCLR + if (TS == ESMF_TimeIntervalAbsValue(TS)) THEN do while (rTime < currTime) rTime = rTime + TCPL end do + else + do while (rTime > currTime) + rTime = rTime + TCPL + end do + end if STATE%TIME_TO_CLEAR(J) = ESMF_AlarmCreate(NAME='TIME2CLEAR_' // trim(COMP_NAME) & // '_' // trim(NAME), & @@ -418,6 +450,29 @@ subroutine Initialize(CC, SRC, DST, CLOCK, RC) rc=STATUS ) _VERIFY(STATUS) + if (clock_debug .and. MAPL_AM_I_ROOT()) THEN + call ESMF_TimeGet ( rTime, timeString=tmpstring, rc=status ) ; _VERIFY(STATUS) + + read(tmpstring( 1: 4),'(i4.4)') year + read(tmpstring( 6: 7),'(i2.2)') month + read(tmpstring( 9:10),'(i2.2)') day + read(tmpstring(12:13),'(i2.2)') hour + read(tmpstring(15:16),'(i2.2)') minute + write(6,1010) & + year, month, day, hour, minute, trim(COMP_NAME)//'_'//trim(NAME) + + call ESMF_TimeIntervalGet ( tcpl, h=hour, m=minute, s=second, rc=status ) ; _VERIFY(STATUS) + write(6,1011) & + hour, minute, second, trim(COMP_NAME)//'_'//trim(NAME) + call ESMF_TimeIntervalGet ( tclr, h=hour, m=minute, s=second, rc=status ) ; _VERIFY(STATUS) + write(6,1011) & + hour, minute, second, 'CLR_'//trim(COMP_NAME)//'_'//trim(NAME) + call ESMF_TimeIntervalGet ( toff, h=hour, m=minute, s=second, rc=status ) ; _VERIFY(STATUS) + write(6,1011) & + hour, minute, second, 'TOFF_'//trim(COMP_NAME)//'_'//trim(NAME) + write(6,*) ' State interval: ', STATE%COUPLE_INTERVAL(J) + endif + if(rTime == currTime) then call ESMF_AlarmRingerOn(STATE%TIME_TO_CLEAR(J), rc=status); _VERIFY(STATUS) end if diff --git a/base/MAPL_HistoryGridComp.F90 b/base/MAPL_HistoryGridComp.F90 index 92e956f52f7..5d4efa64d23 100644 --- a/base/MAPL_HistoryGridComp.F90 +++ b/base/MAPL_HistoryGridComp.F90 @@ -274,10 +274,12 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(ESMF_State), pointer :: export (:) => null() type(ESMF_State), pointer :: exptmp (:) type(ESMF_Time) :: StartTime + type(ESMF_Time) :: EndTime type(ESMF_Time) :: CurrTime type(ESMF_Time) :: RingTime type(ESMF_Time) :: RefTime type(ESMF_TimeInterval) :: Frequency + type(ESMF_TimeInterval) :: OneSecond type(ESMF_Array) :: array type(ESMF_Field) :: field type(ESMF_Field) :: f @@ -423,6 +425,15 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(ESMF_Field), allocatable :: fldList(:) character(len=ESMF_MAXSTR), allocatable :: regexList(:) +! debu nonsense + type(ESMF_Time) :: debugTime + type(ESMF_TimeInterval) :: timeStep + character(len=ESMF_MAXSTR) :: TimeString + logical :: ringing + integer :: alarmCount +#ifdef ADJOINT + integer :: reverseTime +#endif ! Begin !------ @@ -462,6 +473,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGet ( clock, calendar=cal, rc=STATUS ) ; _VERIFY(STATUS) call ESMF_ClockGet ( clock, currTime=CurrTime, rc=STATUS ) ; _VERIFY(STATUS) call ESMF_ClockGet ( clock, StartTime=StartTime,rc=STATUS ) ; _VERIFY(STATUS) + call ESMF_ClockGet ( clock, stopTime=EndTime, rc=STATUS ) ; _VERIFY(STATUS) call ESMF_TimeGet ( StartTime, TimeString=string ,rc=STATUS ) ; _VERIFY(STATUS) read(string( 1: 4),'(i4.4)') year @@ -549,6 +561,15 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute(config, value=intstate%version, & label='VERSION:', default=0, rc=status) _VERIFY(STATUS) + +#ifdef ADJOINT + ! Are we running the adjoint? + call ESMF_ConfigGetAttribute(config, reverseTime, & + Label="REVERSE_TIME:" , & + Default=0, RC=STATUS) + _VERIFY(STATUS) +#endif + if( MAPL_AM_I_ROOT() ) then print * print *, 'EXPSRC:',trim(INTSTATE%expsrc) @@ -559,6 +580,12 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) print *, 'MarkDone: ' , INTSTATE%MarkDone print *, 'PrePost: ' , INTSTATE%PrePost print * +#ifdef ADJOINT + print *, 'Checking if this is adjoint. REVERSE_TIME = "', reverseTime, '"' + if (reverseTime .eq. 1) THEN + print *, 'Yes!' + endif +#endif endif ! Determine Number of Output Streams @@ -792,8 +819,13 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_date, default=nymdc, & label=trim(string) // 'ref_date:',rc=status ) _VERIFY(STATUS) + if (startTime < endTime) THEN call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_time, default=000000, & label=trim(string) // 'ref_time:',rc=status ) + else + call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_time, default=240000, & + label=trim(string) // 'ref_time:',rc=status ) + endif _VERIFY(STATUS) call ESMF_ConfigGetAttribute ( cfg, list(n)%end_date, default=-999, & @@ -857,7 +889,6 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%ref_time < 0 .OR. & list(n)%duration < 0 ) list(n)%disabled = .true. - old_fields_style = .true. ! unless if (intstate%version >= 2) then call ESMF_ConfigGetAttribute ( cfg, value=field_set_name, label=trim(string)//'field_set:', & @@ -1158,7 +1189,11 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) sec = 0 else IntState%average(n) = .true. - sec = MAPL_nsecf(list(n)%acc_interval) / 2 + if (startTime < endTime) then + sec = MAPL_nsecf(list(n)%acc_interval) / 2 + else + sec = -MAPL_nsecf(list(n)%acc_interval) / 2 + endif endif call ESMF_TimeIntervalSet( INTSTATE%STAMPOFFSET(n), S=sec, rc=status ) _VERIFY(STATUS) @@ -1186,24 +1221,108 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! also in this case sec should be set to non-zero sec = MAPL_nsecf( list(n)%frequency ) call ESMF_TimeIntervalSet( Frequency, S=sec, calendar=cal, rc=status ) ; _VERIFY(STATUS) + if (endTime < starttime) Frequency = Frequency * -1 RingTime = RefTime ! Added Logic to eliminate BEG_DATE = cap_restart date problem ! ------------------------------------------------------------ - if (RefTime == startTime) then - RingTime = RefTime + Frequency - end if + if (endTime > startTime) then + if (RefTime == startTime) then + if (list(n)%backwards) then + RingTime = RefTime + else + RingTime = RefTime + Frequency + endif + end if + else + if (RefTime == endTime) then + if (list(n)%backwards) then + RingTime = endTime - Frequency + else + RingTime = startTime + endif + end if + endif ! - if (RingTime < currTime .and. sec /= 0 ) then - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + if (Frequency == ESMF_TimeIntervalAbsValue(Frequency)) then + if (RingTime < currTime .and. sec /= 0 ) then + if (list(n)%backwards) then + RingTime = RingTime - (INT((currTime - RingTime)/frequency)+1)*frequency + else + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + endif + endif + else + if (RingTime > currTime .and. sec /= 0 ) then + if (list(n)%backwards) then + RingTime = RingTime - (INT((currTime - RingTime)/frequency)+1)*frequency + else + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + endif + endif + endif + if(.true. .and. MAPL_AM_I_ROOT() ) then + write(6,*) "Setting history alarm for species ", n + + call ESMF_TimeGet ( RingTime, timeString=tmpstring, rc=status ) ; _VERIFY(STATUS) + + read(tmpstring( 1: 4),'(i4.4)') year + read(tmpstring( 6: 7),'(i2.2)') month + read(tmpstring( 9:10),'(i2.2)') day + read(tmpstring(12:13),'(i2.2)') hour + read(tmpstring(15:16),'(i2.2)') minute + write(6,'(1X,"RingTime: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " backwards:",L1)') & + year, month, day, hour, minute, list(n)%backwards + + call ESMF_ClockGet(clock, currtime=debugtime, timeStep=timestep,rc=status) ; _VERIFY(STATUS) + call ESMF_TimeIntervalGet ( timestep, h=hour,m=minute, rc=status ) ; _VERIFY(STATUS) + + if (hour < 0 .or. minute < 0) then + write(6,'("Clock Timestep = -", i2.2, ":", i2.2)') hour, abs(minute) + else + write(6,'("Clock Timestep = ", i2.2, ":", i2.2)') hour, abs(minute) + end if + call ESMF_TimeGet ( DebugTime, timeString=tmpstring, rc=status ) ; _VERIFY(STATUS) + + read(tmpstring( 1: 4),'(i4.4)') year + read(tmpstring( 6: 7),'(i2.2)') month + read(tmpstring( 9:10),'(i2.2)') day + read(tmpstring(12:13),'(i2.2)') hour + read(tmpstring(15:16),'(i2.2)') minute + write(6,'(1X,"Current Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " backwards:",L1)') & + year, month, day, hour, minute, list(n)%backwards endif if ( list(n)%backwards ) then - list(n)%his_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, rc=status ) + list(n)%his_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) else list(n)%his_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) endif _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + + ! I don't understand what's going on here. For some reason when I create the alarm when the clock is running + ! with a negative timestep, it immediately rings, even though it's an hour in the past... + ! if (ESMF_AlarmIsRinging(list(n)%his_alarm)) THEN + ! call ESMF_AlarmSet(list(n)%his_alarm, RingTime=RingTime, ringing=.false., rc=status); _VERIFY(STATUS) + ! endif + if(.false. .and. MAPL_AM_I_ROOT() ) then + call ESMF_AlarmGet(list(n)%his_alarm, RingTime=DebugTime, ringing=ringing, rc=STATUS); _VERIFY(STATUS) + call ESMF_TimeGet ( DebugTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"Alarm ", i3, " Ring Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " ringing: ", L1)') & + n, year, month, day, hour, minute, ringing + endif + + !ALT if monthly overwrite duration and frequency if (list(n)%monthly) then list(n)%duration = 1 !ALT simply non-zero @@ -1218,15 +1337,23 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! and for debugging print call WRITE_PARALLEL("DEBUG: monthly averaging is active for collection "//trim(list(n)%collection)) end if - if (RingTime < currTime) then - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency - endif - if ( list(n)%backwards ) then - list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, rc=status ) + if (startTime < endTime) THEN + RingTime = RefTime + IntState%StampOffset(n) + if (RingTime < currTime) then + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + endif else - list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) + RingTime = RefTime - IntState%StampOffset(n) + if (RingTime > currTime) then + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency + endif endif + list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) + _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) + WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount if (list(n)%monthly .and. (currTime == RingTime)) then call ESMF_AlarmRingerOn( list(n)%his_alarm,rc=status ) _VERIFY(STATUS) @@ -1237,6 +1364,10 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, enabled=.false., & ringTime=currTime, name='historyNewSegment', rc=status ) _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount endif ! Mon Alarm based on 1st of Month 00Z @@ -1260,11 +1391,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) do while ( RingTime < currTime ) RingTime = RingTime + Frequency enddo - if ( list(n)%backwards ) then - list(n)%mon_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, rc=status ) - else - list(n)%mon_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) - endif + list(n)%mon_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) _VERIFY(STATUS) if(list(n)%monthly) then !ALT this is temporary workaround. It has a memory leak @@ -1274,6 +1401,10 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%his_alarm = list(n)%mon_alarm intState%stampOffset(n) = Frequency ! we go to the beginning of the month end if + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount ! End Alarm based on end_date and end_time ! ---------------------------------------- @@ -1292,19 +1423,30 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) M = REF_TIME(5), & S = REF_TIME(6), calendar=cal, rc=rc ) - if ( list(n)%backwards ) then - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, rc=status ) - else - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, sticky=.false., rc=status ) - endif + list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, sticky=.false., rc=status ) _VERIFY(STATUS) else - if ( list(n)%backwards ) then - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=CurrTime, rc=status ) +#ifdef ADJOINT + if (reverseTime .eq. 1) then + if (MAPL_Am_I_Root()) & + WRITE(*,*) ' Setting end_alarm to disabled!' + call ESMF_TimeIntervalSet( OneSecond, S = 1, rc=rc ) + ringTime = startTime - oneSecond + list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=ringTime, sticky=.false., enabled=.false., rc=status ) else - list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=CurrTime, sticky=.false., rc=status ) +#endif + call ESMF_TimeIntervalSet( OneSecond, S = 1, rc=rc ) + ringTime = endTime + oneSecond + list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=ringTime, sticky=.false., rc=status ) +#ifdef ADJOINT endif +#endif _VERIFY(STATUS) + call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & + alarmCount=alarmCount, rc = status ) + _VERIFY(STATUS) + WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + call ESMF_AlarmRingerOff(list(n)%end_alarm, rc=status ) _VERIFY(STATUS) endif @@ -2167,6 +2309,12 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) else + if (.false. .and. MAPL_AM_I_ROOT()) THEN + WRITE(*,*) 'REFRESH = ', REFRESH, ' AVGINT = ', AVGINT + WRITE(*,*) 'acc_int = ', MAPL_nsecf(list(n)%acc_interval), & + ' freq = ', MAPL_nsecf(list(n)%frequency) + endif + call MAPL_VarSpecCreateInList(INTSTATE%SRCS(n)%SPEC, & SHORT_NAME = SHORT_NAME, & LONG_NAME = LONG_NAME, & @@ -2178,7 +2326,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) FIELD_TYPE = FIELD_TYPE, & RC=STATUS ) _VERIFY(STATUS) - + if (startTime < endTime) THEN call MAPL_VarSpecCreateInList(INTSTATE%DSTS(n)%SPEC, & SHORT_NAME = list(n)%field_set%fields(3,m), & LONG_NAME = LONG_NAME, & @@ -2190,6 +2338,19 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) GRID = GRID, & FIELD_TYPE = FIELD_TYPE, & RC=STATUS ) + else + call MAPL_VarSpecCreateInList(INTSTATE%DSTS(n)%SPEC, & + SHORT_NAME = list(n)%field_set%fields(3,m), & + LONG_NAME = LONG_NAME, & + UNITS = UNITS, & + DIMS = DIMS, & + ACCMLT_INTERVAL= -MAPL_nsecf(list(n)%acc_interval), & + COUPLE_INTERVAL= -MAPL_nsecf(list(n)%frequency ), & + VLOCATION = VLOCATION, & + GRID = GRID, & + FIELD_TYPE = FIELD_TYPE, & + RC=STATUS ) + endif _VERIFY(STATUS) endif ! has_ungrid @@ -2197,8 +2358,13 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) else ! else for if averaged - REFRESH = MAPL_nsecf(list(n)%acc_interval) - AVGINT = MAPL_nsecf( list(n)%frequency ) + if (startTime < endTime) THEN + REFRESH = MAPL_nsecf(list(n)%acc_interval) + AVGINT = MAPL_nsecf( list(n)%frequency ) + else + REFRESH = -MAPL_nsecf(list(n)%acc_interval) + AVGINT = -MAPL_nsecf(list(n)%frequency ) + end if call ESMF_AttributeSet(F, NAME='REFRESH_INTERVAL', VALUE=REFRESH, RC=STATUS) _VERIFY(STATUS) call ESMF_AttributeSet(F, NAME='AVERAGING_INTERVAL', VALUE=AVGINT, RC=STATUS) @@ -3232,6 +3398,15 @@ subroutine Run ( gc, import, export, clock, rc ) ! ErrLog vars integer :: status +! Debug nonsense + type(ESMF_Time) :: CurrTime + character(len=ESMF_MAXSTR) :: TimeString + integer :: year,month,day,hour,minute + logical :: alarmEnabled + type(ESMF_Time) :: RingTime + character(len=ESMF_MAXSTR) :: tmpstring + + !============================================================================= ! Begin... @@ -3271,7 +3446,45 @@ subroutine Run ( gc, import, export, clock, rc ) allocate(Ignore (nlist), stat=status) _VERIFY(STATUS) Ignore = .false. + + if(.true. .and. MAPL_AM_I_ROOT() ) then + write(6,*) "Checking history time" + call ESMF_ClockGet ( clock, currTime=CurrTime ,rc=STATUS ) ; _VERIFY(STATUS) + + call ESMF_TimeGet ( CurrTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"CurrTime: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " FWD:",L1)') & + year, month, day, hour, minute, FWD + do n=1,nlist + call ESMF_AlarmGet(list(n)%his_alarm, ringTime=CurrTime, ringing=alarmEnabled, rc=STATUS); _VERIFY(STATUS) + call ESMF_TimeGet ( CurrTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"Alarm ", i3, " Ring Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " ringing: ", L1)') & + n, year, month, day, hour, minute, alarmEnabled + + call ESMF_AlarmGet(list(n)%his_alarm, prevRingTime=CurrTime, enabled=alarmEnabled, rc=STATUS); _VERIFY(STATUS) + call ESMF_TimeGet ( CurrTime, timeString=TimeString, rc=status ) ; _VERIFY(STATUS) + + read(timestring( 1: 4),'(i4.4)') year + read(timestring( 6: 7),'(i2.2)') month + read(timestring( 9:10),'(i2.2)') day + read(timestring(12:13),'(i2.2)') hour + read(timestring(15:16),'(i2.2)') minute + write(6,'(1X,"Alarm ", i3, " Prev Ring Time: ",i4.4, "/", i2.2, "/", i2.2, "T", i2.2, ":", i2.2, " enabled: ", L1)') & + n, year, month, day, hour, minute, alarmEnabled + end do + endif ! decide if clock direction and collections' backwards mode agree do n=1,nlist @@ -3350,6 +3563,10 @@ subroutine Run ( gc, import, export, clock, rc ) if (list(n)%disabled .or. ESMF_AlarmIsRinging(list(n)%end_alarm) ) then list(n)%disabled = .true. Writing(n) = .false. + if (MAPL_am_I_Root()) THEN + WRITE(*, 1999) n + ENDIF +1999 FORMAT('Not Writing alarm ', i3, ' because end_alarm is ringing') else if (list(n)%timeseries_output) then Writing(n) = .true. else diff --git a/base/MAPL_IO.F90 b/base/MAPL_IO.F90 index 115823ba905..a6cc1c71b50 100644 --- a/base/MAPL_IO.F90 +++ b/base/MAPL_IO.F90 @@ -7453,6 +7453,7 @@ subroutine MAPL_StateVarReadNCPar(filename, STATE, arrdes, bootstrapable, NAME, logical :: bootstrapable_ logical :: isPresent character(len=:), allocatable :: fname_by_face + character(len=ESMF_MAXSTR) :: dbgmsg ! get a list of variables in the file so we can skip if the ! variable in the state is not in the file and it is bootstrapable ! will just let root do this since everybody will need it @@ -7592,9 +7593,18 @@ subroutine MAPL_StateVarReadNCPar(filename, STATE, arrdes, bootstrapable, NAME, else if (bootStrapable_ .and. (RST == MAPL_RestartOptional)) then call WRITE_PARALLEL(" Bootstrapping Variable: "//trim(FieldName)//" in "//trim(filename)) + IF (INDEX('FieldName', 'ADJ') /= 0) THEN + + call WRITE_PARALLEL(" Setting restart atr to : MAPL_RestartOptional") + + ! Set restart attr to indicate bootstrapped + call ESMF_AttributeSet ( field, name='RESTART', & + value=MAPL_RestartOptional, rc=status) + else + call WRITE_PARALLEL(" Setting restart atr to : MAPL_RestartBootstrap") call ESMF_AttributeSet ( field, name='RESTART', & value=MAPL_RestartBootstrap, rc=status) - + endif else _ASSERT(.false., " Could not find field "//trim(FieldName)//" in "//trim(filename)) end if @@ -7620,6 +7630,12 @@ subroutine MAPL_StateVarReadNCPar(filename, STATE, arrdes, bootstrapable, NAME, else RST = MAPL_RestartOptional end if + + if (MAPL_Am_I_Root()) THEN + WRITE(*, 1010) FieldName, RST, MAPL_RestartOptional + endif +1010 format(a10, '%RST = ', i3, ' RstOpt = ', i3) + skipReading = (RST == MAPL_RestartSkip) if (skipReading) cycle call ESMF_AttributeGet(field, name='doNotAllocate', isPresent=isPresent, rc=status) @@ -7647,8 +7663,18 @@ subroutine MAPL_StateVarReadNCPar(filename, STATE, arrdes, bootstrapable, NAME, else if (bootStrapable .and. (RST == MAPL_RestartOptional)) then call WRITE_PARALLEL(" Bootstrapping Variable: "//trim(FieldName)//" in "//trim(filename)) + if (INDEX(FieldName, 'ADJ') /= 0) THEN + call WRITE_PARALLEL(" Setting restart atr to : MAPL_RestartSkipInitial") + + ! Set restart attr to indicate bootstrapped + call ESMF_AttributeSet ( field, name='RESTART', & + value=MAPL_RestartSkipInitial, rc=status) + else + + call WRITE_PARALLEL(" Setting restart atr to : MAPL_RestartBootstrap") call ESMF_AttributeSet ( field, name='RESTART', & value=MAPL_RestartBootstrap, rc=status) + endif else _ASSERT(.false., " Could not find field "//trim(Fieldname)//" in "//trim(filename)) end if @@ -7849,6 +7875,7 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc) type (StringIntegerMap), save :: RstCollections type (StringIntegerMapIterator) :: iter type (StringVariableMap) :: var_map + character(len=ESMF_MAXSTR) :: fname call ESMF_FieldBundleGet(Bundle,FieldCount=nVars, name=BundleName, rc=STATUS) _VERIFY(STATUS) @@ -8023,6 +8050,14 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc) ! add 1 for time ndims = ndims + 1 + ! get rid of - or + sign at the beginning of filename + if (filename(1:1) .eq. '-' .or. filename(1:1) .eq. '+') THEN + FNAME = trim(filename(2:)) + else + FNAME = trim(filename) + end if + + !WJ note: if arrdes%write_restart_by_oserver is true, all processors will participate if (arrdes%writers_comm/=MPI_COMM_NULL .or. arrdes%write_restart_by_oserver) then @@ -8384,13 +8419,13 @@ subroutine MAPL_BundleWriteNCPar(Bundle, arrdes, CLOCK, filename, oClients, rc) else ! not written by oserver if (arrdes%num_writers == 1) then - call formatter%create(trim(filename), rc=status) + call formatter%create(trim(fname), rc=status) _VERIFY(status) call formatter%write(cf,rc=status) _VERIFY(STATUS) else if (arrdes%write_restart_by_face) then - fname_by_face = get_fname_by_face(trim(filename),arrdes%face_index) + fname_by_face = get_fname_by_face(trim(fname),arrdes%face_index) call formatter%create_par(trim(fname_by_face),comm=arrdes%face_writers_comm,info=info,rc=status) _VERIFY(status) else From 32fa5e2bed6d081f2ae4c98745260f9482c0a8c6 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Fri, 16 Apr 2021 15:28:14 -0700 Subject: [PATCH 2/5] Removed debug print statements --- base/MAPL_CapGridComp.F90 | 6 ------ base/MAPL_HistoryGridComp.F90 | 12 ++++++------ base/MAPL_IO.F90 | 6 +++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/base/MAPL_CapGridComp.F90 b/base/MAPL_CapGridComp.F90 index 644991f1524..df205123e10 100644 --- a/base/MAPL_CapGridComp.F90 +++ b/base/MAPL_CapGridComp.F90 @@ -1267,13 +1267,10 @@ subroutine step(this, rc) n=delt64/real(this%heartbeat_dt,kind=real64) ! GridCompRun Timer [Inst] RUN_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_RUN_TIMER-START_RUN_TIMER) - WRITE(*,*) 'RUN_THROUGHPUT = ', RUN_THROUGHPUT ! Time loop throughput [Inst] INST_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER-START_TIMER) - WRITE(*,*) 'INST_THROUGHPUT = ', INST_THROUGHPUT ! Time loop throughput [Avg] LOOP_THROUGHPUT = REAL(n*this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER- this%loop_start_timer) - WRITE(*,*) 'LOOP_THROUGHPUT = ', LOOP_THROUGHPUT ! Estimate time remaining (seconds) TIME_REMAINING = REAL((this%nsteps-n)*this%HEARTBEAT_DT,kind=REAL64)/LOOP_THROUGHPUT HRS_R = FLOOR(TIME_REMAINING/3600.0) @@ -1413,13 +1410,10 @@ subroutine step_reverse(this, first, rc) n=delt64/real(this%heartbeat_dt,kind=real64) ! GridCompRun Timer [Inst] RUN_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_RUN_TIMER-START_RUN_TIMER) - WRITE(*,*) 'RUN_THROUGHPUT = ', RUN_THROUGHPUT ! Time loop throughput [Inst] INST_THROUGHPUT = REAL( this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER-START_TIMER) - WRITE(*,*) 'INST_THROUGHPUT = ', INST_THROUGHPUT ! Time loop throughput [Avg] LOOP_THROUGHPUT = REAL(n*this%HEARTBEAT_DT,kind=REAL64)/(END_TIMER- this%loop_start_timer) - WRITE(*,*) 'LOOP_THROUGHPUT = ', LOOP_THROUGHPUT ! Estimate time remaining (seconds) TIME_REMAINING = REAL((this%nsteps-n)*this%HEARTBEAT_DT,kind=REAL64)/LOOP_THROUGHPUT HRS_R = FLOOR(TIME_REMAINING/3600.0) diff --git a/base/MAPL_HistoryGridComp.F90 b/base/MAPL_HistoryGridComp.F90 index 5d4efa64d23..b197e0117ce 100644 --- a/base/MAPL_HistoryGridComp.F90 +++ b/base/MAPL_HistoryGridComp.F90 @@ -1302,7 +1302,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) - WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount ! I don't understand what's going on here. For some reason when I create the alarm when the clock is running ! with a negative timestep, it immediately rings, even though it's an hour in the past... @@ -1353,7 +1353,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) - WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount if (list(n)%monthly .and. (currTime == RingTime)) then call ESMF_AlarmRingerOn( list(n)%his_alarm,rc=status ) _VERIFY(STATUS) @@ -1367,7 +1367,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) - WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount endif ! Mon Alarm based on 1st of Month 00Z @@ -1404,7 +1404,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) - WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount ! End Alarm based on end_date and end_time ! ---------------------------------------- @@ -1445,7 +1445,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & alarmCount=alarmCount, rc = status ) _VERIFY(STATUS) - WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount + ! WRITE(6,'(" History Clock now has ", i3, " alarms.")') alarmCount call ESMF_AlarmRingerOff(list(n)%end_alarm, rc=status ) _VERIFY(STATUS) @@ -3447,7 +3447,7 @@ subroutine Run ( gc, import, export, clock, rc ) _VERIFY(STATUS) Ignore = .false. - if(.true. .and. MAPL_AM_I_ROOT() ) then + if(.false. .and. MAPL_AM_I_ROOT() ) then write(6,*) "Checking history time" call ESMF_ClockGet ( clock, currTime=CurrTime ,rc=STATUS ) ; _VERIFY(STATUS) diff --git a/base/MAPL_IO.F90 b/base/MAPL_IO.F90 index a6cc1c71b50..e6385217673 100644 --- a/base/MAPL_IO.F90 +++ b/base/MAPL_IO.F90 @@ -7631,9 +7631,9 @@ subroutine MAPL_StateVarReadNCPar(filename, STATE, arrdes, bootstrapable, NAME, RST = MAPL_RestartOptional end if - if (MAPL_Am_I_Root()) THEN - WRITE(*, 1010) FieldName, RST, MAPL_RestartOptional - endif + ! if (MAPL_Am_I_Root()) THEN + ! WRITE(*, 1010) FieldName, RST, MAPL_RestartOptional + ! endif 1010 format(a10, '%RST = ', i3, ' RstOpt = ', i3) skipReading = (RST == MAPL_RestartSkip) From ffe4eff889a28200197b918031a405459518f22e Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 20 Apr 2021 12:13:09 -0700 Subject: [PATCH 3/5] Removed remnants of abandonned attempt. I originally made an attempt at running the reverse integration by using a negative timestemp, but this required more code changes and more runtime configuration changes, so that was abandonned in favour of using the REVERSE_TIME resource. However, I didn't roll all those changes back when I changed strategies so there were a number of those code changes left hanging around. I have now removed them and confirmed that their removal does not impact the forward or reverse results. --- base/MAPL_CapGridComp.F90 | 33 ++++++------ base/MAPL_Generic.F90 | 25 +++------ base/MAPL_GenericCplComp.F90 | 9 +--- base/MAPL_HistoryGridComp.F90 | 95 ++++++++--------------------------- 4 files changed, 44 insertions(+), 118 deletions(-) diff --git a/base/MAPL_CapGridComp.F90 b/base/MAPL_CapGridComp.F90 index df205123e10..2088a435d2c 100644 --- a/base/MAPL_CapGridComp.F90 +++ b/base/MAPL_CapGridComp.F90 @@ -1013,6 +1013,7 @@ subroutine run_MAPL_GridComp(gc, rc) ! name, loop counter, result code character (len=ESMF_MAXSTR) :: name integer :: i, result + integer :: istop cap => get_CapGridComp_from_gc(gc) call MAPL_GetObjectFromGC(gc, maplobj, rc=status) @@ -1049,7 +1050,6 @@ subroutine run_MAPL_GridComp(gc, rc) ! called properly on reverse time run ! Time Loop starts by checking for Segment Ending Time !----------------------------------------------------- - if (.true.) then FAKE_TIME_LOOP: do n = 1, cap%nsteps if (MAPL_Am_I_Root()) & @@ -1078,7 +1078,7 @@ subroutine run_MAPL_GridComp(gc, rc) ! Update the cap_restart_time variable to avoid crash on last timestep call ESMF_ClockGetNextTime(cap%clock,nextTime=cap%cap_restart_time,rc=status) _VERIFY(status) - endif + if (MAPL_Am_I_Root()) & WRITE(*,*) ' MAPL_CapGC finished time loop, reversing clocks' @@ -1094,9 +1094,15 @@ subroutine run_MAPL_GridComp(gc, rc) call ESMF_ClockSet ( cap%clock_hist, direction=ESMF_DIRECTION_REVERSE, & advancecount=0, rc=status ) _VERIFY(STATUS) - endif + endif if (.not. cap%printspec > 0) then + ! Reverse loop needs an extra initialization step + if (.not. reverse_time) then + istop = cap%nsteps + else + istop = cap%nsteps + 1 + endif ! Time Loop starts by checking for Segment Ending Time !----------------------------------------------------- @@ -1104,7 +1110,7 @@ subroutine run_MAPL_GridComp(gc, rc) _VERIFY(status) cap%loop_start_timer = MPI_WTime(status) cap%started_loop_timer = .true. - TIME_LOOP: do n = 1, cap%nsteps+1 + TIME_LOOP: do n = 1, istop call MAPL_MemUtilsWrite(cap%vm, 'MAPL_Cap:TimeLoop', rc = status) _VERIFY(status) @@ -1127,17 +1133,11 @@ subroutine run_MAPL_GridComp(gc, rc) if (done) exit endif -#ifdef ADJOINT - if (.not. reverse_time) THEN -#endif - call cap%step(status) -#ifdef ADJOINT - ELSE - if (MAPL_Am_I_Root()) & - print *, 'Calling step_reverse' - call cap%step_reverse(n .eq. 1, status) - ENDIF -#endif + if (.not. reverse_time) THEN + call cap%step(status) + ELSE + call cap%step_reverse(n .eq. 1, status) + ENDIF _VERIFY(status) ! Reset loop average timer to get a better @@ -1295,7 +1295,6 @@ subroutine step(this, rc) _RETURN(ESMF_SUCCESS) end subroutine step -#ifdef ADJOINT subroutine step_reverse(this, first, rc) class(MAPL_CapGridComp), intent(inout) :: this logical, intent(in) :: first @@ -1437,7 +1436,7 @@ subroutine step_reverse(this, first, rc) _RETURN(ESMF_SUCCESS) end subroutine step_reverse -#endif + ! !IROUTINE: MAPL_ClockInit -- Sets the clock ! !INTERFACE: diff --git a/base/MAPL_Generic.F90 b/base/MAPL_Generic.F90 index 134cb16c491..c433d590565 100644 --- a/base/MAPL_Generic.F90 +++ b/base/MAPL_Generic.F90 @@ -1279,30 +1279,17 @@ recursive subroutine MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, RC ) H = HH, M = MM, S = SS, rc = STATUS ) _VERIFY(STATUS) - if (timeint == ESMF_TimeIntervalAbsValue(timeint)) then - if (ringTime > currTime) then - ringTime = ringTime - (INT((ringTime - currTime)/TIMEINT)+1)*TIMEINT - end if - else - if (ringTime < currTime) then - ringTime = ringTime - (INT((ringTime - currTime)/TIMEINT)+1)*TIMEINT - end if - endif + if (ringTime > currTime) then + ringTime = ringTime - (INT((ringTime - currTime)/TIMEINT)+1)*TIMEINT + end if ringTime = ringTime-TSTEP ! we back off current time with clock's dt since ! we advance the clock AFTER run method - if (timeint == ESMF_TimeIntervalAbsValue(timeint)) then - ! make sure that ringTime is not in the past - do while (ringTime < currTime) - ringTime = ringTime + TIMEINT - end do - else ! make sure that ringTime is not in the past - do while (ringTime > currTime) - ringTime = ringTime + TIMEINT - end do - endif + do while (ringTime < currTime) + ringTime = ringTime + TIMEINT + end do STATE%ALARM(0) = ESMF_AlarmCreate(CLOCK = CLOCK, & name = trim(COMP_NAME) // "_Alarm" , & diff --git a/base/MAPL_GenericCplComp.F90 b/base/MAPL_GenericCplComp.F90 index 63d59d09c22..e75375fdb83 100644 --- a/base/MAPL_GenericCplComp.F90 +++ b/base/MAPL_GenericCplComp.F90 @@ -426,20 +426,13 @@ subroutine Initialize(CC, SRC, DST, CLOCK, RC) calendar=cal, RC=STATUS) _VERIFY(STATUS) - if (ESMF_TimeIntervalAbsValue(TCLR) < ESMF_TimeIntervalAbsValue(TS)) & - TCLR = TS + if (TCLR < TS) TCLR = TS rTime = TM0 + TOFF - TCLR - if (TS == ESMF_TimeIntervalAbsValue(TS)) THEN do while (rTime < currTime) rTime = rTime + TCPL end do - else - do while (rTime > currTime) - rTime = rTime + TCPL - end do - end if STATE%TIME_TO_CLEAR(J) = ESMF_AlarmCreate(NAME='TIME2CLEAR_' // trim(COMP_NAME) & // '_' // trim(NAME), & diff --git a/base/MAPL_HistoryGridComp.F90 b/base/MAPL_HistoryGridComp.F90 index b197e0117ce..a68ecd1f5f8 100644 --- a/base/MAPL_HistoryGridComp.F90 +++ b/base/MAPL_HistoryGridComp.F90 @@ -819,13 +819,10 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_date, default=nymdc, & label=trim(string) // 'ref_date:',rc=status ) _VERIFY(STATUS) - if (startTime < endTime) THEN + call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_time, default=000000, & label=trim(string) // 'ref_time:',rc=status ) - else - call ESMF_ConfigGetAttribute ( cfg, list(n)%ref_time, default=240000, & - label=trim(string) // 'ref_time:',rc=status ) - endif + _VERIFY(STATUS) call ESMF_ConfigGetAttribute ( cfg, list(n)%end_date, default=-999, & @@ -1189,11 +1186,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) sec = 0 else IntState%average(n) = .true. - if (startTime < endTime) then - sec = MAPL_nsecf(list(n)%acc_interval) / 2 - else - sec = -MAPL_nsecf(list(n)%acc_interval) / 2 - endif + sec = MAPL_nsecf(list(n)%acc_interval) / 2 endif call ESMF_TimeIntervalSet( INTSTATE%STAMPOFFSET(n), S=sec, rc=status ) _VERIFY(STATUS) @@ -1221,44 +1214,23 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! also in this case sec should be set to non-zero sec = MAPL_nsecf( list(n)%frequency ) call ESMF_TimeIntervalSet( Frequency, S=sec, calendar=cal, rc=status ) ; _VERIFY(STATUS) - if (endTime < starttime) Frequency = Frequency * -1 RingTime = RefTime ! Added Logic to eliminate BEG_DATE = cap_restart date problem ! ------------------------------------------------------------ - if (endTime > startTime) then - if (RefTime == startTime) then - if (list(n)%backwards) then - RingTime = RefTime - else - RingTime = RefTime + Frequency - endif - end if - else - if (RefTime == endTime) then - if (list(n)%backwards) then - RingTime = endTime - Frequency - else - RingTime = startTime - endif - end if - endif -! - if (Frequency == ESMF_TimeIntervalAbsValue(Frequency)) then - if (RingTime < currTime .and. sec /= 0 ) then - if (list(n)%backwards) then - RingTime = RingTime - (INT((currTime - RingTime)/frequency)+1)*frequency - else - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency - endif + if (RefTime == startTime) then + if (list(n)%backwards) then + RingTime = RefTime + else + RingTime = RefTime + Frequency endif - else - if (RingTime > currTime .and. sec /= 0 ) then - if (list(n)%backwards) then - RingTime = RingTime - (INT((currTime - RingTime)/frequency)+1)*frequency - else - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency - endif + end if +! + if (RingTime < currTime .and. sec /= 0 ) then + if (list(n)%backwards) then + RingTime = RingTime - (INT((currTime - RingTime)/frequency)+1)*frequency + else + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency endif endif if(.true. .and. MAPL_AM_I_ROOT() ) then @@ -1337,16 +1309,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ! and for debugging print call WRITE_PARALLEL("DEBUG: monthly averaging is active for collection "//trim(list(n)%collection)) end if - if (startTime < endTime) THEN - RingTime = RefTime + IntState%StampOffset(n) - if (RingTime < currTime) then - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency - endif - else - RingTime = RefTime - IntState%StampOffset(n) - if (RingTime > currTime) then - RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency - endif + RingTime = RefTime + IntState%StampOffset(n) + if (RingTime < currTime) then + RingTime = RingTime + (INT((currTime - RingTime)/frequency)+1)*frequency endif list(n)%seg_alarm = ESMF_AlarmCreate( clock=clock, RingInterval=Frequency, RingTime=RingTime, sticky=.false., rc=status ) _VERIFY(STATUS) @@ -2326,7 +2291,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) FIELD_TYPE = FIELD_TYPE, & RC=STATUS ) _VERIFY(STATUS) - if (startTime < endTime) THEN + call MAPL_VarSpecCreateInList(INTSTATE%DSTS(n)%SPEC, & SHORT_NAME = list(n)%field_set%fields(3,m), & LONG_NAME = LONG_NAME, & @@ -2338,19 +2303,6 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) GRID = GRID, & FIELD_TYPE = FIELD_TYPE, & RC=STATUS ) - else - call MAPL_VarSpecCreateInList(INTSTATE%DSTS(n)%SPEC, & - SHORT_NAME = list(n)%field_set%fields(3,m), & - LONG_NAME = LONG_NAME, & - UNITS = UNITS, & - DIMS = DIMS, & - ACCMLT_INTERVAL= -MAPL_nsecf(list(n)%acc_interval), & - COUPLE_INTERVAL= -MAPL_nsecf(list(n)%frequency ), & - VLOCATION = VLOCATION, & - GRID = GRID, & - FIELD_TYPE = FIELD_TYPE, & - RC=STATUS ) - endif _VERIFY(STATUS) endif ! has_ungrid @@ -2358,13 +2310,8 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) else ! else for if averaged - if (startTime < endTime) THEN - REFRESH = MAPL_nsecf(list(n)%acc_interval) - AVGINT = MAPL_nsecf( list(n)%frequency ) - else - REFRESH = -MAPL_nsecf(list(n)%acc_interval) - AVGINT = -MAPL_nsecf(list(n)%frequency ) - end if + REFRESH = MAPL_nsecf(list(n)%acc_interval) + AVGINT = MAPL_nsecf( list(n)%frequency ) call ESMF_AttributeSet(F, NAME='REFRESH_INTERVAL', VALUE=REFRESH, RC=STATUS) _VERIFY(STATUS) call ESMF_AttributeSet(F, NAME='AVERAGING_INTERVAL', VALUE=AVGINT, RC=STATUS) From a335b5501ae8dbe6913f6d509ecaf4b57f603f72 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 20 Apr 2021 12:17:46 -0700 Subject: [PATCH 4/5] Removed accidentally committed debug setting --- base/MAPL_CapOptions.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/MAPL_CapOptions.F90 b/base/MAPL_CapOptions.F90 index 46ec07dae2e..51eca157e2e 100644 --- a/base/MAPL_CapOptions.F90 +++ b/base/MAPL_CapOptions.F90 @@ -16,7 +16,7 @@ module MAPL_CapOptionsMod logical :: use_comm_world = .true. character(:), allocatable :: egress_file character(:), allocatable :: cap_rc_file - type (ESMF_LogKind_Flag) :: esmf_logging_mode = ESMF_LOGKIND_SINGLE + type (ESMF_LogKind_Flag) :: esmf_logging_mode = ESMF_LOGKIND_NONE integer :: npes_model = -1 ! only one of the next two options can have nonzero values integer, allocatable :: npes_input_server(:) From 9ab91102a31c0b192a42fd3fb948c07caa502f34 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 26 Apr 2021 16:54:36 -0700 Subject: [PATCH 5/5] Removed last ifdef ADJOINT tags and improved some comments --- base/MAPL_CapGridComp.F90 | 2 -- base/MAPL_GenericCplComp.F90 | 2 +- base/MAPL_HistoryGridComp.F90 | 19 +++++-------------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/base/MAPL_CapGridComp.F90 b/base/MAPL_CapGridComp.F90 index 2088a435d2c..0c9f0476c33 100644 --- a/base/MAPL_CapGridComp.F90 +++ b/base/MAPL_CapGridComp.F90 @@ -60,9 +60,7 @@ module MAPL_CapGridCompMod procedure :: initialize_history procedure :: run procedure :: step -#ifdef ADJOINT procedure :: step_reverse -#endif procedure :: finalize procedure :: get_model_duration procedure :: get_am_i_root diff --git a/base/MAPL_GenericCplComp.F90 b/base/MAPL_GenericCplComp.F90 index e75375fdb83..8189d9ca360 100644 --- a/base/MAPL_GenericCplComp.F90 +++ b/base/MAPL_GenericCplComp.F90 @@ -282,7 +282,7 @@ subroutine Initialize(CC, SRC, DST, CLOCK, RC) type(ESMF_Field) :: field integer :: cplfunc logical :: isPresent -! debug nonsense +! debug variables logical, parameter :: clock_debug = .false. character(len=ESMF_MAXSTR) :: tmpstring integer :: year,month,day,hour,minute,second diff --git a/base/MAPL_HistoryGridComp.F90 b/base/MAPL_HistoryGridComp.F90 index a68ecd1f5f8..d96611ead52 100644 --- a/base/MAPL_HistoryGridComp.F90 +++ b/base/MAPL_HistoryGridComp.F90 @@ -425,15 +425,14 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(ESMF_Field), allocatable :: fldList(:) character(len=ESMF_MAXSTR), allocatable :: regexList(:) -! debu nonsense +! debug variables type(ESMF_Time) :: debugTime type(ESMF_TimeInterval) :: timeStep character(len=ESMF_MAXSTR) :: TimeString logical :: ringing integer :: alarmCount -#ifdef ADJOINT +! adjoint variable integer :: reverseTime -#endif ! Begin !------ @@ -562,13 +561,11 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) label='VERSION:', default=0, rc=status) _VERIFY(STATUS) -#ifdef ADJOINT ! Are we running the adjoint? call ESMF_ConfigGetAttribute(config, reverseTime, & Label="REVERSE_TIME:" , & Default=0, RC=STATUS) _VERIFY(STATUS) -#endif if( MAPL_AM_I_ROOT() ) then print * @@ -580,12 +577,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) print *, 'MarkDone: ' , INTSTATE%MarkDone print *, 'PrePost: ' , INTSTATE%PrePost print * -#ifdef ADJOINT - print *, 'Checking if this is adjoint. REVERSE_TIME = "', reverseTime, '"' if (reverseTime .eq. 1) THEN - print *, 'Yes!' + print *, 'REVERSE_TIME = "', reverseTime, '"' endif -#endif endif ! Determine Number of Output Streams @@ -1391,7 +1385,6 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=RingTime, sticky=.false., rc=status ) _VERIFY(STATUS) else -#ifdef ADJOINT if (reverseTime .eq. 1) then if (MAPL_Am_I_Root()) & WRITE(*,*) ' Setting end_alarm to disabled!' @@ -1399,13 +1392,11 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) ringTime = startTime - oneSecond list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=ringTime, sticky=.false., enabled=.false., rc=status ) else -#endif call ESMF_TimeIntervalSet( OneSecond, S = 1, rc=rc ) ringTime = endTime + oneSecond list(n)%end_alarm = ESMF_AlarmCreate( clock=clock, RingTime=ringTime, sticky=.false., rc=status ) -#ifdef ADJOINT endif -#endif + _VERIFY(STATUS) call ESMF_ClockGetAlarmList(clock, ESMF_ALARMLIST_ALL, & alarmCount=alarmCount, rc = status ) @@ -3345,7 +3336,7 @@ subroutine Run ( gc, import, export, clock, rc ) ! ErrLog vars integer :: status -! Debug nonsense +! Debug variables type(ESMF_Time) :: CurrTime character(len=ESMF_MAXSTR) :: TimeString integer :: year,month,day,hour,minute