From 93cd922c9d4e5feb99aa09c11fc0c46cc3d14b00 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 13 Oct 2020 11:48:15 -0700 Subject: [PATCH 01/11] Added template files to geos-chem run directory --- .gitmodules | 2 +- src/GCHP_GridComp/GEOSChem_GridComp/geos-chem | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 68bd7d23d..676ff8717 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,7 +18,7 @@ url = https://github.com/geoschem/FVdycoreCubed_GridComp.git [submodule "src/GCHP_GridComp/GEOSChem_GridComp/geos-chem"] path = src/GCHP_GridComp/GEOSChem_GridComp/geos-chem - url = https://github.com/geoschem/geos-chem.git + url = https://github.com/TerribleNews/geos-chem.git [submodule "src/GCHP_GridComp/HEMCO_GridComp/HEMCO"] path = src/GCHP_GridComp/HEMCO_GridComp/HEMCO url = https://github.com/geoschem/HEMCO.git diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index ecedda016..7515c064b 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit ecedda0167b63820b9eed888f812804edc96e45e +Subproject commit 7515c064b2cdda6de953ead4edd87201e8e95bac From d5353017127c8668776eadd3a84f93098e458cea Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Fri, 4 Dec 2020 12:08:00 -0800 Subject: [PATCH 02/11] Forward and reverse models compile and run. --- CMakeLists.txt | 6 + src/GCHP_GridComp/FVdycoreCubed_GridComp | 2 +- src/GCHP_GridComp/GCHP_GridCompMod.F90 | 258 +++++++++++++++++- .../GCHPctmEnv_GridCompMod.F90 | 21 ++ src/GCHP_GridComp/GEOSChem_GridComp/geos-chem | 2 +- src/GCHP_GridComp/HEMCO_GridComp/HEMCO | 2 +- src/MAPL | 2 +- 7 files changed, 287 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 185267572..25821107d 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,3 +106,9 @@ set_target_properties( PROPERTIES EXCLUDE_FROM_ALL TRUE ) + +target_compile_definitions(Baselibs INTERFACE + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> + "" +) diff --git a/src/GCHP_GridComp/FVdycoreCubed_GridComp b/src/GCHP_GridComp/FVdycoreCubed_GridComp index 4ff5e9249..d87af81f4 160000 --- a/src/GCHP_GridComp/FVdycoreCubed_GridComp +++ b/src/GCHP_GridComp/FVdycoreCubed_GridComp @@ -1 +1 @@ -Subproject commit 4ff5e9249dc51c19c84fbda9bb1a13e7e630912d +Subproject commit d87af81f4244b5cda3d8da13b10933db075c7583 diff --git a/src/GCHP_GridComp/GCHP_GridCompMod.F90 b/src/GCHP_GridComp/GCHP_GridCompMod.F90 index 16fb3406f..9b2c15ad4 100644 --- a/src/GCHP_GridComp/GCHP_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHP_GridCompMod.F90 @@ -91,6 +91,10 @@ subroutine SetServices ( GC, RC ) type (ESMF_Config) :: CF logical :: am_I_Root +#ifdef ADJOINT + character(len=ESMF_MAXSTR) :: ModelPhase + logical :: isAdjoint +#endif !============================================================================= @@ -115,7 +119,24 @@ subroutine SetServices ( GC, RC ) RC=STATUS ) _VERIFY(STATUS) -!BOP +#ifdef ADJOINT + CALL ESMF_ConfigGetAttribute( CF, ModelPhase, & + Label = "MODEL_PHASE:",& + Default="FORWARD", & + __RC__ ) + isAdjoint = .false. + if (trim(ModelPhase) == 'ADJOINT') & + isAdjoint = .true. + +#ifdef REVERSE_OPERATORS + IF (.not. isAdjoint) THEN + IF (MAPL_Am_I_Root()) & + WRITE(*,*) ' Forward run, adding children in standard order. ' // ModelPhase +#else + WRITE(*,*) ' Adding children in standard order. MODEL_PHASE: ' // ModelPhase +#endif + +#endif ! !IMPORT STATE: @@ -139,6 +160,29 @@ subroutine SetServices ( GC, RC ) RC=STATUS) _VERIFY(STATUS) +#ifdef ADJOINT +#ifdef REVERSE_OPERATORS + ELSE + IF (MAPL_Am_I_Root()) & + WRITE(*,*) ' Adjoint run, adding children in reverse order. ' + ! Add dynamics + ADV = MAPL_AddChild(GC, NAME='DYNAMICS', SS=AtmosAdvSetServices, & + RC=STATUS) + _VERIFY(STATUS) + + ! Add chemistry + CHEM = MAPL_AddChild(GC, NAME='GCHPchem', SS=AtmosChemSetServices, & + RC=STATUS) + _VERIFY(STATUS) + + ! Add component for deriving variables for other components + ECTM = MAPL_AddChild(GC, NAME='GCHPctmEnv' , SS=EctmSetServices, & + RC=STATUS) + _VERIFY(STATUS) + ENDIF +#endif +#endif + ! Set internal connections between the children`s IMPORTS and EXPORTS ! ------------------------------------------------------------------- !BOP @@ -274,11 +318,23 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GridCreate( GC, rc=status ) _VERIFY(STATUS) + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Before Generic Init' + endif + + ! Try to locate a grid cell. This should only ever work on one PET + + + ! Call Initialize for every Child !-------------------------------- call MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, __RC__ ) _VERIFY(STATUS) + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'After Generic Init' + endif + call MAPL_TimerOn(STATE,"TOTAL") ! call MAPL_TimerOn(STATE,"INITIALIZE") @@ -353,7 +409,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: I, L integer :: IM, JM, LM real :: DT - + + character(len=ESMF_MAXSTR) :: ModelPhase + logical :: isAdjoint + logical, save :: firstRun = .true. + !============================================================================= ! Begin... @@ -377,6 +437,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) _VERIFY(STATUS) endif +#ifdef ADJOINT + ! Check if this is an adjoint run + CALL ESMF_ConfigGetAttribute( CF, ModelPhase, & + Label = "MODEL_PHASE:",& + Default="FORWARD", & + __RC__ ) + isAdjoint = .false. + if (trim(ModelPhase) == 'ADJOINT') & + isAdjoint = .true. +#endif + + ! Start timers !------------- call MAPL_TimerOn(STATE,"TOTAL") @@ -401,6 +473,21 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) call ESMF_ConfigGetAttribute(CF, DT, Label="RUN_DT:" , RC=STATUS) _VERIFY(STATUS) +#ifdef ADJOINT + !------------------------------------------------------------ + ! If we're doing the adoint, we should be running these in + ! reverse order. Possibly not the Environment module? + ! In cany case, this isn't working yet, so disable it for now + !------------------------------------------------------------ +#ifdef REVERSE_OPERATORS + IF (.not. isAdjoint) THEN +#else + IF (.true.) THEN +#endif + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Not reversing high-level operators' + endif +#endif ! Cinderella Component: to derive variables for other components !--------------------- @@ -493,10 +580,177 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) 'GCHP, after GEOS-Chem: ', RC=STATUS ) _VERIFY(STATUS) endif +#ifdef ADJOINT + ELSE + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Reversing high-level operators' + endif + + IF (firstRun) THEN + ! Cinderella Component: to derive variables for other components + !--------------------- + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ECTM) ) + call ESMF_GridCompRun ( GCS(ECTM), & + importState = GIM(ECTM), & + exportState = GEX(ECTM), & + clock = CLOCK, & + userRC = STATUS ) + _VERIFY(STATUS) + + call MAPL_TimerOff( STATE, GCNames(ECTM) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Dynamics & Advection + !------------------ + ! SDE 2017-02-18: This needs to run even if transport is off, as it is + ! responsible for the pressure level edge arrays. It already has an internal + ! switch ("AdvCore_Advection") which can be used to prevent any actual + ! transport taking place by bypassing the advection calculation. + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ADV) ) + call ESMF_GridCompRun ( GCS(ADV), & + importState = GIM(ADV), & + exportState = GEX(ADV), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff( STATE, GCNames(ADV) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ENDIF + ! Chemistry + !------------------ + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS-Chem: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(CHEM) ) + call ESMF_GridCompRun ( GCS(CHEM), & + importState = GIM(CHEM), & + exportState = GEX(CHEM), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, CHEM, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff(STATE,GCNames(CHEM)) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS-Chem: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Cinderella Component: to derive variables for other components + !--------------------- + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ECTM) ) + call ESMF_GridCompRun ( GCS(ECTM), & + importState = GIM(ECTM), & + exportState = GEX(ECTM), & + clock = CLOCK, & + userRC = STATUS ) + _VERIFY(STATUS) + + call MAPL_TimerOff( STATE, GCNames(ECTM) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Dynamics & Advection + !------------------ + ! SDE 2017-02-18: This needs to run even if transport is off, as it is + ! responsible for the pressure level edge arrays. It already has an internal + ! switch ("AdvCore_Advection") which can be used to prevent any actual + ! transport taking place by bypassing the advection calculation. + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + call MAPL_TimerOn ( STATE, GCNames(ADV) ) + call ESMF_GridCompRun ( GCS(ADV), & + importState = GIM(ADV), & + exportState = GEX(ADV), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff( STATE, GCNames(ADV) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ENDIF +#endif call MAPL_TimerOff(STATE,"RUN") call MAPL_TimerOff(STATE,"TOTAL") + firstRun = .false. + _RETURN(ESMF_SUCCESS) end subroutine Run diff --git a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 index 4d2d34ca7..dbdae2641 100755 --- a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 @@ -541,6 +541,12 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: i, j real(r8) :: PSDry0, PSDry1, PEdge_Bot, PEdge_Top + logical, save :: firstRun = .true. + +#ifdef ADJOINT + integer :: reverseTime +#endif + ! Get the target components name and set-up traceback handle. ! ----------------------------------------------------------- call ESMF_GridCompGet ( GC, name=COMP_NAME, Grid=esmfGrid, RC=STATUS ) @@ -561,6 +567,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) _VERIFY(STATUS) dt = ndt +#ifdef ADJOINT + call MAPL_GetResource( ggState, reverseTime, 'REVERSE_TIME:', default=0, RC=STATUS ) + _VERIFY(STATUS) + IF ( MAPL_Am_I_Root() ) THEN + WRITE(*,*) ' GIGCenv REVERSE_TIME: ', reverseTime + ENDIF + IF ( reverseTime .eq. 1) THEN + WRITE(*,*) ' GIGCenv swapping timestep sign.' + dt = -dt + ENDIF +#endif + ! Get to the imports... ! --------------------- call MAPL_GetPointer ( IMPORT, PS0, 'PS1', RC=STATUS ) @@ -676,8 +694,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) ! Use dry pressure at the start of the timestep to calculate mass ! fluxes. GMAO method uses mid-step UC, VC and PLE? PLEr8 = 1.00d0*(DryPLE0r8) + if (.not. firstRun) THEN call fv_computeMassFluxes(UCr8, VCr8, PLEr8, & MFXr8, MFYr8, CXr8, CYr8, dt) + endif + firstRun = .false. !DEALLOCATE( UCr8, VCr8, PLEr8, PLE0, PLE1, DryPLE0, DryPLE1 ) DEALLOCATE( UCr8, VCr8, PLEr8, UC, VC) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index 7515c064b..e6e2ebd1c 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit 7515c064b2cdda6de953ead4edd87201e8e95bac +Subproject commit e6e2ebd1c069aa5c325f8168bead56de7a156128 diff --git a/src/GCHP_GridComp/HEMCO_GridComp/HEMCO b/src/GCHP_GridComp/HEMCO_GridComp/HEMCO index a252fa266..057363277 160000 --- a/src/GCHP_GridComp/HEMCO_GridComp/HEMCO +++ b/src/GCHP_GridComp/HEMCO_GridComp/HEMCO @@ -1 +1 @@ -Subproject commit a252fa26611965cfbeb8352b6e962ab3f644df0e +Subproject commit 057363277fc0820ee7d4828ccbd23c4308b9992b diff --git a/src/MAPL b/src/MAPL index 7821bdd92..020d7e403 160000 --- a/src/MAPL +++ b/src/MAPL @@ -1 +1 @@ -Subproject commit 7821bdd9247b09077f0755717c8051c03fe64700 +Subproject commit 020d7e4038af9a5dbcbf9c4bc38771ca76be1c97 From b9ef5b717434d98bec28f338094c1860c70a9bf4 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 7 Dec 2020 08:24:11 -0800 Subject: [PATCH 03/11] Updated create run directory for adjoint --- src/GCHP_GridComp/GEOSChem_GridComp/geos-chem | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index e6e2ebd1c..83d328c29 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit e6e2ebd1c069aa5c325f8168bead56de7a156128 +Subproject commit 83d328c296b26227e99fddebdddfb76142544716 From 79642a4c195e2587e9e79e60c07eca19b7a81da8 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 7 Dec 2020 09:27:50 -0800 Subject: [PATCH 04/11] Trying to fix submodule branch problem --- src/GCHP_GridComp/GEOSChem_GridComp/geos-chem | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index 83d328c29..cd488b8f7 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit 83d328c296b26227e99fddebdddfb76142544716 +Subproject commit cd488b8f73a4f81cb1052cb4042e140242c19150 From 725b93ddc3142a70a494490d5bb609e39cd1f01e Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Fri, 4 Dec 2020 12:08:00 -0800 Subject: [PATCH 05/11] Forward and reverse models compile and run. --- CMakeLists.txt | 6 + src/GCHP_GridComp/GCHP_GridCompMod.F90 | 258 +++++++++++++++++- .../GCHPctmEnv_GridCompMod.F90 | 21 ++ 3 files changed, 283 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52092f3dd..397fb66d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,3 +115,9 @@ set_target_properties( PROPERTIES EXCLUDE_FROM_ALL TRUE ) + +target_compile_definitions(Baselibs INTERFACE + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> + "" +) diff --git a/src/GCHP_GridComp/GCHP_GridCompMod.F90 b/src/GCHP_GridComp/GCHP_GridCompMod.F90 index 16fb3406f..9b2c15ad4 100644 --- a/src/GCHP_GridComp/GCHP_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHP_GridCompMod.F90 @@ -91,6 +91,10 @@ subroutine SetServices ( GC, RC ) type (ESMF_Config) :: CF logical :: am_I_Root +#ifdef ADJOINT + character(len=ESMF_MAXSTR) :: ModelPhase + logical :: isAdjoint +#endif !============================================================================= @@ -115,7 +119,24 @@ subroutine SetServices ( GC, RC ) RC=STATUS ) _VERIFY(STATUS) -!BOP +#ifdef ADJOINT + CALL ESMF_ConfigGetAttribute( CF, ModelPhase, & + Label = "MODEL_PHASE:",& + Default="FORWARD", & + __RC__ ) + isAdjoint = .false. + if (trim(ModelPhase) == 'ADJOINT') & + isAdjoint = .true. + +#ifdef REVERSE_OPERATORS + IF (.not. isAdjoint) THEN + IF (MAPL_Am_I_Root()) & + WRITE(*,*) ' Forward run, adding children in standard order. ' // ModelPhase +#else + WRITE(*,*) ' Adding children in standard order. MODEL_PHASE: ' // ModelPhase +#endif + +#endif ! !IMPORT STATE: @@ -139,6 +160,29 @@ subroutine SetServices ( GC, RC ) RC=STATUS) _VERIFY(STATUS) +#ifdef ADJOINT +#ifdef REVERSE_OPERATORS + ELSE + IF (MAPL_Am_I_Root()) & + WRITE(*,*) ' Adjoint run, adding children in reverse order. ' + ! Add dynamics + ADV = MAPL_AddChild(GC, NAME='DYNAMICS', SS=AtmosAdvSetServices, & + RC=STATUS) + _VERIFY(STATUS) + + ! Add chemistry + CHEM = MAPL_AddChild(GC, NAME='GCHPchem', SS=AtmosChemSetServices, & + RC=STATUS) + _VERIFY(STATUS) + + ! Add component for deriving variables for other components + ECTM = MAPL_AddChild(GC, NAME='GCHPctmEnv' , SS=EctmSetServices, & + RC=STATUS) + _VERIFY(STATUS) + ENDIF +#endif +#endif + ! Set internal connections between the children`s IMPORTS and EXPORTS ! ------------------------------------------------------------------- !BOP @@ -274,11 +318,23 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GridCreate( GC, rc=status ) _VERIFY(STATUS) + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Before Generic Init' + endif + + ! Try to locate a grid cell. This should only ever work on one PET + + + ! Call Initialize for every Child !-------------------------------- call MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, __RC__ ) _VERIFY(STATUS) + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'After Generic Init' + endif + call MAPL_TimerOn(STATE,"TOTAL") ! call MAPL_TimerOn(STATE,"INITIALIZE") @@ -353,7 +409,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: I, L integer :: IM, JM, LM real :: DT - + + character(len=ESMF_MAXSTR) :: ModelPhase + logical :: isAdjoint + logical, save :: firstRun = .true. + !============================================================================= ! Begin... @@ -377,6 +437,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) _VERIFY(STATUS) endif +#ifdef ADJOINT + ! Check if this is an adjoint run + CALL ESMF_ConfigGetAttribute( CF, ModelPhase, & + Label = "MODEL_PHASE:",& + Default="FORWARD", & + __RC__ ) + isAdjoint = .false. + if (trim(ModelPhase) == 'ADJOINT') & + isAdjoint = .true. +#endif + + ! Start timers !------------- call MAPL_TimerOn(STATE,"TOTAL") @@ -401,6 +473,21 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) call ESMF_ConfigGetAttribute(CF, DT, Label="RUN_DT:" , RC=STATUS) _VERIFY(STATUS) +#ifdef ADJOINT + !------------------------------------------------------------ + ! If we're doing the adoint, we should be running these in + ! reverse order. Possibly not the Environment module? + ! In cany case, this isn't working yet, so disable it for now + !------------------------------------------------------------ +#ifdef REVERSE_OPERATORS + IF (.not. isAdjoint) THEN +#else + IF (.true.) THEN +#endif + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Not reversing high-level operators' + endif +#endif ! Cinderella Component: to derive variables for other components !--------------------- @@ -493,10 +580,177 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) 'GCHP, after GEOS-Chem: ', RC=STATUS ) _VERIFY(STATUS) endif +#ifdef ADJOINT + ELSE + if (MAPL_Am_I_Root()) THEN + WRITE(*,*) 'Reversing high-level operators' + endif + + IF (firstRun) THEN + ! Cinderella Component: to derive variables for other components + !--------------------- + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ECTM) ) + call ESMF_GridCompRun ( GCS(ECTM), & + importState = GIM(ECTM), & + exportState = GEX(ECTM), & + clock = CLOCK, & + userRC = STATUS ) + _VERIFY(STATUS) + + call MAPL_TimerOff( STATE, GCNames(ECTM) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Dynamics & Advection + !------------------ + ! SDE 2017-02-18: This needs to run even if transport is off, as it is + ! responsible for the pressure level edge arrays. It already has an internal + ! switch ("AdvCore_Advection") which can be used to prevent any actual + ! transport taking place by bypassing the advection calculation. + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ADV) ) + call ESMF_GridCompRun ( GCS(ADV), & + importState = GIM(ADV), & + exportState = GEX(ADV), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff( STATE, GCNames(ADV) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ENDIF + ! Chemistry + !------------------ + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS-Chem: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(CHEM) ) + call ESMF_GridCompRun ( GCS(CHEM), & + importState = GIM(CHEM), & + exportState = GEX(CHEM), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, CHEM, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff(STATE,GCNames(CHEM)) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS-Chem: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Cinderella Component: to derive variables for other components + !--------------------- + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + call MAPL_TimerOn ( STATE, GCNames(ECTM) ) + call ESMF_GridCompRun ( GCS(ECTM), & + importState = GIM(ECTM), & + exportState = GEX(ECTM), & + clock = CLOCK, & + userRC = STATUS ) + _VERIFY(STATUS) + + call MAPL_TimerOff( STATE, GCNames(ECTM) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after GEOS_ctmE: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ! Dynamics & Advection + !------------------ + ! SDE 2017-02-18: This needs to run even if transport is off, as it is + ! responsible for the pressure level edge arrays. It already has an internal + ! switch ("AdvCore_Advection") which can be used to prevent any actual + ! transport taking place by bypassing the advection calculation. + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, before Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + call MAPL_TimerOn ( STATE, GCNames(ADV) ) + call ESMF_GridCompRun ( GCS(ADV), & + importState = GIM(ADV), & + exportState = GEX(ADV), & + clock = CLOCK, & + userRC = STATUS ); + _VERIFY(STATUS) + call MAPL_GenericRunCouplers (STATE, ADV, CLOCK, RC=STATUS ); + _VERIFY(STATUS) + call MAPL_TimerOff( STATE, GCNames(ADV) ) + + if ( MemDebugLevel > 0 ) THEN + call ESMF_VMBarrier(VM, RC=STATUS) + _VERIFY(STATUS) + call MAPL_MemUtilsWrite(VM, & + 'GIGC, after Advection: ', RC=STATUS ) + _VERIFY(STATUS) + endif + + ENDIF +#endif call MAPL_TimerOff(STATE,"RUN") call MAPL_TimerOff(STATE,"TOTAL") + firstRun = .false. + _RETURN(ESMF_SUCCESS) end subroutine Run diff --git a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 index 4d2d34ca7..dbdae2641 100755 --- a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 @@ -541,6 +541,12 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: i, j real(r8) :: PSDry0, PSDry1, PEdge_Bot, PEdge_Top + logical, save :: firstRun = .true. + +#ifdef ADJOINT + integer :: reverseTime +#endif + ! Get the target components name and set-up traceback handle. ! ----------------------------------------------------------- call ESMF_GridCompGet ( GC, name=COMP_NAME, Grid=esmfGrid, RC=STATUS ) @@ -561,6 +567,18 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) _VERIFY(STATUS) dt = ndt +#ifdef ADJOINT + call MAPL_GetResource( ggState, reverseTime, 'REVERSE_TIME:', default=0, RC=STATUS ) + _VERIFY(STATUS) + IF ( MAPL_Am_I_Root() ) THEN + WRITE(*,*) ' GIGCenv REVERSE_TIME: ', reverseTime + ENDIF + IF ( reverseTime .eq. 1) THEN + WRITE(*,*) ' GIGCenv swapping timestep sign.' + dt = -dt + ENDIF +#endif + ! Get to the imports... ! --------------------- call MAPL_GetPointer ( IMPORT, PS0, 'PS1', RC=STATUS ) @@ -676,8 +694,11 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) ! Use dry pressure at the start of the timestep to calculate mass ! fluxes. GMAO method uses mid-step UC, VC and PLE? PLEr8 = 1.00d0*(DryPLE0r8) + if (.not. firstRun) THEN call fv_computeMassFluxes(UCr8, VCr8, PLEr8, & MFXr8, MFYr8, CXr8, CYr8, dt) + endif + firstRun = .false. !DEALLOCATE( UCr8, VCr8, PLEr8, PLE0, PLE1, DryPLE0, DryPLE1 ) DEALLOCATE( UCr8, VCr8, PLEr8, UC, VC) From 45c5176a76abb20e2e878a0790d54af1b01b19eb Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Wed, 30 Dec 2020 16:01:03 -0800 Subject: [PATCH 06/11] Added adjoint flags to cmake options --- src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt | 2 ++ src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt b/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt index e6bb932e0..86da2082d 100755 --- a/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt +++ b/src/GCHP_GridComp/GEOSChem_GridComp/CMakeLists.txt @@ -22,6 +22,8 @@ add_subdirectory(geos-chem EXCLUDE_FROM_ALL) target_compile_definitions(GEOSChemBuildProperties INTERFACE ESMF_ EXTERNAL_GRID NC_HAS_COMPRESSION MODEL_GCHPCTM MODEL_GCHP $<$:USE_REAL8> $<$:RRTMG> + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> ) target_link_libraries(GEOSChemBuildProperties INTERFACE diff --git a/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt b/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt index 0a2cf04c9..82e7ca677 100644 --- a/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt +++ b/src/GCHP_GridComp/HEMCO_GridComp/CMakeLists.txt @@ -1,3 +1,9 @@ # HEMCO_GridComp/CMakeLists.txt add_subdirectory(HEMCO EXCLUDE_FROM_ALL) + +target_compile_definitions(HEMCOBuildProperties INTERFACE + $<$:ADJOINT> + $<$:REVERSE_OPERATORS> + "" +) From 4b48f392211b29401388870aaf98cc834010e65f Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 8 Mar 2021 16:57:16 -0800 Subject: [PATCH 07/11] Fixed CO2 run dir in GEOS-Chem submodule --- src/GCHP_GridComp/GEOSChem_GridComp/geos-chem | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index f358e6a30..81a3a0eab 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit f358e6a3097657ac2179801a934ba0e71eab1eab +Subproject commit 81a3a0eabb807caeba933e71178c9fc782cc69f1 From b8501c8d658657ec8ebd38ca614b2f46e059c171 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Tue, 9 Mar 2021 16:12:01 -0800 Subject: [PATCH 08/11] Moved submodules to forked versions --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7679f7aec..1dd2a4748 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "src/MAPL"] path = src/MAPL - url = https://github.com/geoschem/MAPL + url = https://github.com/TerribleNews/MAPL [submodule "src/GMAO_Shared"] path = src/GMAO_Shared url = https://github.com/geoschem/GMAO_Shared @@ -15,13 +15,13 @@ url = https://github.com/geoschem/FMS.git [submodule "src/GCHP_GridComp/FVdycoreCubed_GridComp"] path = src/GCHP_GridComp/FVdycoreCubed_GridComp - url = https://github.com/geoschem/FVdycoreCubed_GridComp.git + url = https://github.com/TerribleNews/FVdycoreCubed_GridComp.git [submodule "src/GCHP_GridComp/GEOSChem_GridComp/geos-chem"] path = src/GCHP_GridComp/GEOSChem_GridComp/geos-chem url = https://github.com/TerribleNews/geos-chem.git [submodule "src/GCHP_GridComp/HEMCO_GridComp/HEMCO"] path = src/GCHP_GridComp/HEMCO_GridComp/HEMCO - url = https://github.com/geoschem/HEMCO.git + url = https://github.com/TerribleNews/HEMCO.git [submodule "src/yaFyaml"] path = src/yaFyaml url = https://github.com/geoschem/yaFyaml.git From e1142215a33cc6ceca858a6d4223202112300e61 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 29 Mar 2021 17:38:23 -0700 Subject: [PATCH 09/11] Updated run files to match MPI configuration of runConfig.sh --- src/GCHP_GridComp/GEOSChem_GridComp/geos-chem | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem index 81a3a0eab..5891c0a91 160000 --- a/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem +++ b/src/GCHP_GridComp/GEOSChem_GridComp/geos-chem @@ -1 +1 @@ -Subproject commit 81a3a0eabb807caeba933e71178c9fc782cc69f1 +Subproject commit 5891c0a91d88e27ef9d25a6222418111fb019bb1 From e53eb7014e2dca7b76d4d1f336fc1be000b1a95e Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 5 Apr 2021 11:22:07 -0700 Subject: [PATCH 10/11] Wrapped debug printing in ifdef ADJOINT blocks --- src/GCHP_GridComp/GCHP_GridCompMod.F90 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/GCHP_GridComp/GCHP_GridCompMod.F90 b/src/GCHP_GridComp/GCHP_GridCompMod.F90 index 9b2c15ad4..da4be39fe 100644 --- a/src/GCHP_GridComp/GCHP_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHP_GridCompMod.F90 @@ -318,9 +318,11 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GridCreate( GC, rc=status ) _VERIFY(STATUS) +#ifdef ADJOINT if (MAPL_Am_I_Root()) THEN WRITE(*,*) 'Before Generic Init' endif +#endif ! Try to locate a grid cell. This should only ever work on one PET @@ -331,9 +333,11 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC ) call MAPL_GenericInitialize ( GC, IMPORT, EXPORT, CLOCK, __RC__ ) _VERIFY(STATUS) +#ifdef ADJOINT if (MAPL_Am_I_Root()) THEN WRITE(*,*) 'After Generic Init' endif +#endif call MAPL_TimerOn(STATE,"TOTAL") ! call MAPL_TimerOn(STATE,"INITIALIZE") From 3685f4e6e94fda148e3aae62801d9e83248fe639 Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Wed, 7 Apr 2021 11:44:29 -0700 Subject: [PATCH 11/11] Moved adjoint-only code into ifdef ADJOINT block --- .../GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 index dbdae2641..af026acdd 100755 --- a/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 +++ b/src/GCHP_GridComp/GCHPctmEnv_GridComp/GCHPctmEnv_GridCompMod.F90 @@ -541,9 +541,8 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: i, j real(r8) :: PSDry0, PSDry1, PEdge_Bot, PEdge_Top - logical, save :: firstRun = .true. - #ifdef ADJOINT + logical, save :: firstRun = .true. integer :: reverseTime #endif @@ -694,11 +693,15 @@ subroutine Run ( GC, IMPORT, EXPORT, CLOCK, RC ) ! Use dry pressure at the start of the timestep to calculate mass ! fluxes. GMAO method uses mid-step UC, VC and PLE? PLEr8 = 1.00d0*(DryPLE0r8) +#ifdef ADJOINT if (.not. firstRun) THEN +#endif call fv_computeMassFluxes(UCr8, VCr8, PLEr8, & MFXr8, MFYr8, CXr8, CYr8, dt) +#ifdef ADJOINT endif firstRun = .false. +#endif !DEALLOCATE( UCr8, VCr8, PLEr8, PLE0, PLE1, DryPLE0, DryPLE1 ) DEALLOCATE( UCr8, VCr8, PLEr8, UC, VC)