C $Header: /u/gcmpack/MITgcm/model/src/the_model_main.F,v 1.117 2014/08/15 20:41:50 jmc Exp $
C $Name: $
CBOI
C
C !TITLE: MITGCM KERNEL CODE SYNOPSIS
C !AUTHORS: mitgcm developers ( support@mitgcm.org )
C !AFFILIATION: Massachussetts Institute of Technology
C !DATE:
C !INTRODUCTION: Kernel dynamical routines
C This document summarises MITgcm code under the model/ subdirectory.
C The code under model/ ( src/ and inc/ ) contains most of
C the driver routines for the baseline forms of the kernel equations in the
C MITgcm algorithm. Numerical code for much of the baseline forms of
C these equations is also under the model/ directory. Other numerical code
C used for the kernel equations is contained in packages in the pkg/
C directory tree.
C Code for auxiliary equations and alternate discretizations of the kernel
C equations and algorithm can also be found in the pkg/ directory tree.
C
C \subsection{Getting Help and Reporting Errors and Problems}
C If you have questions please subscribe and e-mail support@mitgcm.org.
C We also welcome reports of errors and inconsistencies in the code or
C in the accompanying documentation. Please feel free to send these
C to support@mitgcm.org. For further information and to review
C problems reported to support@mitgcm.org please visit http://mitgcm.org.
C
C \subsection{MITgcm Kernel Code Calling Sequence}
C \bv
C
C Invocation from WRAPPER level...
C
C |
C |-THE_MODEL_MAIN :: Primary driver for the MITgcm algorithm
C | :: Called from WRAPPER level numerical
C | :: code invocation routine. On entry
C | :: to THE_MODEL_MAIN separate thread and
C | :: separate processes will have been established.
C | :: Each thread and process will have a unique ID
C | :: but as yet it will not be associated with a
C | :: specific region in decomposed discrete space.
C |
C |-INITIALISE_FIXED :: Set fixed model arrays such as topography,
C | | :: grid, solver matrices etc..
C | |
C | |-INI_PARMS :: Routine to set kernel model parameters.
C | | :: Kernel parameters are read from file "data"
C | | :: in directory in which code executes.
C | |
C | |-PACAKGES_BOOT :: Start up the optional package environment.
C | | :: Runtime selection of active packages.
C | |-PACKAGES_READPARMS :: read all packages input parameter file
C | | |- ${PKG}_READPARMS
C | |
C | |-INI_MODEL_IO :: Initialise Input/Ouput setting
C | |
C | |-INI_GRID :: Control grid array (vert. and hori.) initialisation.
C | | :: Grid arrays are held and described in GRID.h.
C | |
C | |-INI_DEPTHS :: Read (from "bathyFile") or set bathymetry/orography.
C | |-INI_MASKS_ETC :: Derive horizontal and vertical cell fractions and
C | | :: land masking for solid-fluid boundaries.
C | |
C | |-PACKAGES_INIT_FIXED :: do all packages fixed-initialisation setting
C | | |- ${PKG}_INIT_FIXED
C | |
C | |-CONFIG_SUMMARY :: Provide synopsis of kernel setup. Includes
C | | :: annotated table of kernel parameter settings.
C | |
C | |-PACKAGES_CHECK :: call each package configuration checking S/R
C | | |- ${PKG}_CHECK
C | |
C | |-CONFIG_CHECK :: Check config and parameter consistency.
C |
C |-CTRL_UNPACK :: Control vector support package. see pkg/ctrl
C |
C |-ADTHE_MAIN_LOOP :: Derivative evaluating form of main time stepping loop
C ! :: Automatically generated by TAMC/TAF.
C |
C |-THE_MAIN_LOOP :: Main timestepping loop routine.
C | |
C | |-INITIALISE_VARIA :: Set the initial conditions for time evolving
C | | |-INI_DYNVARS :: set common block variable to zero
C | | |-INI_NH_VARS :: set common block variable to zero
C | | |
C | | |-INI_FIELDS :: Control initialising model fields to non-zero
C | | | |-INI_VEL,_THETA,_SALT,_PSURF, ...
C | | | |-READ_PICKUP
C | | |
C | | |-INI_FORCING :: initialise forcing fields
C | | |
C | | |-PACKAGES_INIT_VARIABLES :: Does initialisation of time evolving
C | | | | ${PKG}_INIT_VARIA :: package data.
C | | |
C | | |-MONITOR :: Monitor state (see pkg/monitor)
C | | |-STATE_SUMMARY :: Summarise model prognostic variables.
C | | |-DO_THE_MODEL_IO :: Standard diagnostic I/O.
C | |
C====|>| ****************************
C====|>| BEGIN MAIN TIMESTEPPING LOOP
C====|>| ****************************
C | |-COST_AVERAGESFIELDS :: time-averaged Cost function terms (see pkg/cost)
C | |
C/\ | |-FORWARD_STEP :: Step forward a time-step ( AT LAST !!! )
C/\ | | |
C/\ | | |-LOAD_FIELDS_DRIVER :: control loading of input fields from files
C/\ | | |
C/\ | | |-CPL_EXPORT_MY_DATA :: Send coupling fields to coupler
C/\ | | |-CPL_IMPORT_EXTERNAL_DATA :: Receive coupling fields from coupler
C/\ | | |
C/\ | | |-DO_ATMOSPHERIC_PHYS :: Atmospheric physics computation
C/\ | | |
C/\ | | |-DO_OCEANIC_PHYS :: Oceanic (& seaice) physics computation
C/\ | | | |-OBCS_CALC :: Open boundary. package (see pkg/obcs).
C/\ | | | |-EXTERNAL_FORCING_SURF:: Accumulates appropriately dimensioned
C/\ | | | | :: forcing terms.
C/\ | | |
C/\ | | |-GCHEM_CALC_TENDENCY :: geochemistry driver routine (see pkg/gchem)
C/\ | | |
C/\ | | |-THERMODYNAMICS :: (synchronous time-stepping)
C/\ | | | theta, salt + tracer equations driver.
C/\ | | | |-GMREDI_RESIDUAL_FLOW :: Get the flow field used to advect tracer
C/\ | | | |
C/\ | | | |-TEMP_INTEGRATE :: Step forward Prognostic Eq for Temperature
C/\ | | | | |-GAD_ADVECTION :: Generalised advection driver (multi-dim
C/\ | | | | | advection case) (see pkg/gad).
C/\ | | | | |-GAD_CALC_RHS :: Calculate Advection-Diffusion tendency terms
C/\ | | | | |-TIMESTEP_TRACER :: Step tracer field forward in time
C/\ | | | | |-IMPLDIFF :: Solve vertical implicit diffusion equation.
C/\ | | | | |-CYCLE_AB_TRACER :: Cycle time-stepping arrays for tracer field
C/\ | | | |
C/\ | | | |-SALT_INTEGRATE :: Step forward Prognostic Eq for Salinity
C/\ | | | |
C/\ | | | |-PTRACERS_INTEGRATE :: Integrate other tracer(s) (see pkg/ptracers).
C/\ | | | |
C/\ | | | |-OBCS_APPLY_TS :: Open boundary package (see pkg/obcs ).
C/\ | | |
C/\ | | |-DYNAMICS :: Momentum equations driver.
C/\ | | | |
C/\ | | | |-CALC_GRAD_PHI_SURF :: Calculate the gradient of the surface
C/\ | | | | Potential anomaly.
C/\ | | | |-CALC_VISCOSITY :: Calculate net vertical viscosity
C/\ | | | |-CALC_PHI_HYD :: Integrate the hydrostatic relation.
C/\ | | | |-MOM_FLUXFORM :: Flux form mom eqn. package (pkg/mom_fluxform)
C/\ | | | |-MOM_VECINV :: Vector invariant form mom eqn (pkg/mom_vecinv)
C/\ | | | |-TIMESTEP :: Step momentum fields forward in time
C/\ | | | |-OBCS_APPLY_UV :: Open boundary package (see pkg/obcs).
C/\ | | | |-IMPLDIFF :: Solve vertical implicit diffusion equation.
C/\ | | | |-CALC_GW :: vert. momentum tendency terms (Non-Hydrostatic)
C/\ | | |
C/\ | | |-UPDATE_SURF_DR :: Update the surface-level thickness fraction.
C/\ | | |-UPDATE_R_STAR :: Update the level thickness fraction.
C/\ | | |-UPDATE_CG2D :: Update 2d conjugate grad. for Free-Surf.
C/\ | | |
C/\ | | |-SOLVE_FOR_PRESSURE :: Find surface pressure.
C/\ | | | |-CG2D :: Two-dim pre-con. conjugate-gradient.
C/\ | | | |-CG3D :: Three-dim pre-con. conjugate-gradient solver.
C/\ | | |
C/\ | | |-MOMENTUM_CORRECTION_STEP :: Finalise momentum stepping
C/\ | | | |-CALC_GRAD_PHI_SURF :: Return DDx and DDy of surface pressure
C/\ | | | |-CORRECTION_STEP :: Pressure correction to momentum
C/\ | | | |-OBCS_APPLY_UV :: Open boundary package (see pkg/obcs).
C/\ | | |
C/\ | | |-INTEGR_CONTINUITY :: Integrate continuity equation
C/\ | | |
C/\ | | |-THERMODYNAMICS :: (staggered time-stepping)
C/\ | | | theta, salt + tracer equations driver.
C/\ | | |
C/\ | | |-TRACERS_CORRECTION_STEP :: Finalise tracer steppin:
C/\ | | | apply filter, conv.adjustment
C/\ | | |
C/\ | | |-GCHEM_FORCING_SEP :: tracer forcing for gchem pkg (if tracer
C/\ | | | dependent tendencies calculated separately)
C/\ | | |
C/\ | | |-DO_FIELDS_BLOCKING_EXCHANGES :: Sync up overlap regions.
C/\ | | |
C/\ | | |-MONITOR :: Monitor package (pkg/monitor).
C/\ | | |-DO_THE_MODEL_IO :: Standard diagnostic I/O.
C/\ | | |
C/\ | | |-DO_WRITE_PICKUP :: Write restart files.
C | |
C<===|=| **************************
C<===|=| END MAIN TIMESTEPPING LOOP
C<===|=| **************************
C | |
C | |-COST_AVERAGESFIELDS :: time-averaged Cost function terms (see pkg/cost)
C | |-COST_FINAL :: Cost function package. (see pkg/cost)
C |
C |-CTRL_PACK :: Control vector support package. see pkg/ctrl
C |
C |-GRDCHK_MAIN :: Gradient check package. see pkg/grdchk
C |
C |-TIMER_PRINTALL :: Computational timing summary
C |
C |-COMM_STATS :: Summarise inter-proc and inter-thread communication
C :: events.
C \ev
C
CEOI
#include "PACKAGES_CONFIG.h"
#include "CPP_OPTIONS.h"
#include "AD_CONFIG.h"
#ifdef ALLOW_AUTODIFF
# include "AUTODIFF_OPTIONS.h"
#endif
#ifdef ALLOW_CTRL
# include "CTRL_OPTIONS.h"
#endif
#ifdef ALLOW_STREAMICE
# include "STREAMICE_OPTIONS.h"
#endif
CBOP
C !ROUTINE: THE_MODEL_MAIN
C !INTERFACE:
SUBROUTINE THE_MODEL_MAIN(myThid)
C !DESCRIPTION: \bv
C *==========================================================*
C | SUBROUTINE THE_MODEL_MAIN
C | o Master controlling routine for model using the MITgcm
C | UV parallel wrapper.
C *==========================================================*
C | THE_MODEL_MAIN is invoked by the MITgcm UV parallel
C | wrapper with a single integer argument "myThid". This
C | variable identifies the thread number of an instance of
C | THE_MODEL_MAIN. Each instance of THE_MODEL_MAIN works
C | on a particular region of the models domain and
C | synchronises with other instances as necessary. The
C | routine has to "understand" the MITgcm parallel
C | environment and the numerical algorithm. Editing this
C | routine is best done with some knowledge of both aspects.
C | Notes
C | =====
C | C*P* comments indicating place holders for which code is
C | presently being developed.
C *==========================================================*
C \ev
C !CALLING SEQUENCE:
C THE_MODEL_MAIN()
C |
C |
C |--INITIALISE_FIXED
C | o Set model configuration (fixed arrays)
C | Topography, hydrography, timestep, grid, etc..
C |
C |--CTRL_UNPACK o Derivative mode. Unpack control vector.
C |
C |--ADTHE_MAIN_LOOP o Main timestepping loop for combined
C | prognostic and reverse mode integration.
C |
C |--THE_MAIN_LOOP o Main timestepping loop for pure prognostic
C | integration.
C |
C |--CTRL_PACK o Derivative mode. Unpack control vector.
C |
C |--GRDCHK_MAIN o Gradient check control routine.
C |
C |--TIMER_PRINTALL o Print out timing statistics.
C |
C |--COMM_STATS o Print out communication statistics.
C !USES:
IMPLICIT NONE
C == Global variables ===
#include "SIZE.h"
#include "EEPARAMS.h"
#include "PARAMS.h"
#include "DYNVARS.h"
#ifdef ALLOW_AUTODIFF
# include "tamc.h"
#endif
#ifdef ALLOW_CTRL
# include "ctrl.h"
# include "optim.h"
#endif
C !INPUT/OUTPUT PARAMETERS:
C == Routine arguments ==
C myThid :: Thread number for this instance of the routine.
INTEGER myThid
C !LOCAL VARIABLES:
C == Local variables ==
C Note: Under the multi-threaded model myIter and myTime are local
C variables passed around as routine arguments.
C Although this is fiddly it saves the need to impose
C additional synchronisation points when they are updated.
C myTime :: Time counter for this thread
C myIter :: Iteration counter for this thread
INTEGER myIter
_RL myTime
LOGICAL exst
LOGICAL lastdiva
CEOP
C-- set default:
exst = .TRUE.
lastdiva = .TRUE.
#ifdef ALLOW_PETSC
call STREAMICE_INITIALIZE_PETSC
#endif
#ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_ENTER('THE_MODEL_MAIN',myThid)
#endif
#if defined(USE_PAPI) defined(USE_PCL_FLOPS_SFP) defined(USE_PCL_FLOPS) defined(USE_PCL)
CALL TIMER_CONTROL('','INIT','THE_MODEL_MAIN',myThid)
#endif
C-- This timer encompasses the whole code
CALL TIMER_START('ALL [THE_MODEL_MAIN]',myThid)
#ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('INITIALISE_FIXED',myThid)
#endif
C-- Set model configuration (fixed arrays)
CALL TIMER_START('INITIALISE_FIXED [THE_MODEL_MAIN]',myThid)
CALL INITIALISE_FIXED( myThid )
CALL TIMER_STOP ('INITIALISE_FIXED [THE_MODEL_MAIN]',myThid)
myTime = startTime
myIter = nIter0
#if ( defined (ALLOW_ADMTLM) )
STOP 'should never get here; ADMTLM_DSVD calls ADMTLM_DRIVER'
#elif ( defined (ALLOW_AUTODIFF))
# ifndef EXCLUDE_CTRL_PACK
inquire( file='costfinal', exist=exst )
IF ( .NOT. exst ) THEN
IF ( (optimcycle.NE.0 .OR. .NOT.doinitxx)
& .AND. doMainUnpack ) THEN
CALL TIMER_START('CTRL_UNPACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_UNPACK( .TRUE. , myThid )
CALL TIMER_STOP ('CTRL_UNPACK [THE_MODEL_MAIN]',myThid)
ENDIF
ENDIF
# endif /* EXCLUDE_CTRL_PACK */
# ifdef ALLOW_COST
CALL COST_DEPENDENT_INIT ( myThid )
# endif
# if ( defined (ALLOW_TANGENTLINEAR_RUN) )
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('G_THE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('G_THE_MAIN_LOOP [TANGENT RUN]',myThid)
CALL G_THE_MAIN_LOOP ( myTime, myIter, myThid )
CALL TIMER_STOP ('G_THE_MAIN_LOOP [TANGENT RUN]',myThid)
# elif ( defined (ALLOW_ADJOINT_RUN)
defined (ALLOW_ECCO_OPTIMIZATION) )
# ifdef ALLOW_DIVIDED_ADJOINT
C-- The following assumes the TAF option '-pure'
inquire( file='costfinal', exist=exst )
IF ( .NOT. exst) THEN
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('MDTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('MDTHE_MAIN_LOOP [MD RUN]', myThid)
CALL MDTHE_MAIN_LOOP ( myTime, myIter, myThid )
CALL TIMER_STOP ('MDTHE_MAIN_LOOP [MD RUN]', myThid)
CALL COST_FINAL_STORE ( myThid, lastdiva )
ELSE
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL ADTHE_MAIN_LOOP ( myThid )
CALL TIMER_STOP ('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL COST_FINAL_RESTORE ( myThid, lastdiva )
ENDIF
else /* ALLOW_DIVIDED_ADJOINT undef */
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ADTHE_MAIN_LOOP',myThid)
# endif
CALL TIMER_START('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
CALL ADTHE_MAIN_LOOP ( myThid )
CALL TIMER_STOP ('ADTHE_MAIN_LOOP [ADJOINT RUN]', myThid)
# endif /* ALLOW_DIVIDED_ADJOINT */
else /* forward run only within AD setting */
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('THE_MAIN_LOOP',myThid)
# endif
C-- Call time stepping loop of full model
CALL TIMER_START('THE_MAIN_LOOP [THE_MODEL_MAIN]',myThid)
CALL THE_MAIN_LOOP( myTime, myIter, myThid )
CALL TIMER_STOP ('THE_MAIN_LOOP [THE_MODEL_MAIN]',myThid)
# endif /* forward run only within AD setting */
# ifndef EXCLUDE_CTRL_PACK
IF ( lastdiva .AND. doMainPack ) THEN
CALL TIMER_START('CTRL_PACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_PACK( .FALSE. , myThid )
CALL TIMER_STOP ('CTRL_PACK [THE_MODEL_MAIN]',myThid)
IF ( ( optimcycle.EQ.0 .OR. (.NOT. doMainUnpack) )
& .AND. myIter.EQ.nIter0 ) THEN
CALL TIMER_START('CTRL_PACK [THE_MODEL_MAIN]',myThid)
CALL CTRL_PACK( .TRUE. , myThid )
CALL TIMER_STOP ('CTRL_PACK [THE_MODEL_MAIN]',myThid)
ENDIF
ENDIF
# endif /* EXCLUDE_CTRL_PACK */
# ifdef ALLOW_GRDCHK
IF ( useGrdchk .AND. lastdiva ) THEN
CALL TIMER_START('GRDCHK_MAIN [THE_MODEL_MAIN]',myThid)
CALL GRDCHK_MAIN( myThid )
CALL TIMER_STOP ('GRDCHK_MAIN [THE_MODEL_MAIN]',myThid)
ENDIF
# endif
#else /* ALL AD-related undef */
# ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('THE_MAIN_LOOP',myThid)
# endif
C-- Call time stepping loop of full model
CALL TIMER_START('THE_MAIN_LOOP [THE_MODEL_MAIN]',myThid)
CALL THE_MAIN_LOOP( myTime, myIter, myThid )
CALL TIMER_STOP ('THE_MAIN_LOOP [THE_MODEL_MAIN]',myThid)
#endif /* ALLOW_TANGENTLINEAR_RUN ALLOW_ADJOINT_RUN ALLOW_ADMTLM */
#ifdef ALLOW_PETSC
call STREAMICE_FINALIZE_PETSC
#endif
#ifdef ALLOW_MNC
IF (useMNC) THEN
C Close all open NetCDF files
_BEGIN_MASTER( myThid )
CALL MNC_FILE_CLOSE_ALL( myThid )
_END_MASTER( myThid )
ENDIF
#endif
C-- This timer encompasses the whole code
CALL TIMER_STOP ('ALL [THE_MODEL_MAIN]',myThid)
C-- Write timer statistics
IF ( myThid .EQ. 1 ) THEN
CALL TIMER_PRINTALL( myThid )
CALL COMM_STATS
ENDIF
C-- Check threads synchronization :
CALL BAR_CHECK( 9, myThid )
#ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_LEAVE('THE_MODEL_MAIN',myThid)
#endif
RETURN
END