C $Header: /u/gcmpack/MITgcm/eesupp/inc/EESUPPORT.h,v 1.10 2009/04/21 16:00:53 jmc Exp $
C $Name: $
CBOP
C !ROUTINE: EESUPPORT.h
C !INTERFACE:
C include "EESUPPORT.h"
C
C !DESCRIPTION:
C *==========================================================*
C | EESUPPORT.h |
C *==========================================================*
C | Support data structures for the MITgcm UV ``execution |
C | environment'' code. This data should be private to the |
C | execution environment routines. Data which needs to be |
C | accessed directly by a numerical model goes in |
C | EEPARAMS.h. |
C *==========================================================*
CEOP
C ERROR_HEADER - String which prefixes error messages
CHARACTER*(*) ERROR_HEADER
PARAMETER ( ERROR_HEADER = ' *** ERROR ***' )
C PROCESS_HEADER - String which prefixes processor number
CHARACTER*(*) PROCESS_HEADER
PARAMETER ( PROCESS_HEADER = 'PID.TID' )
C MAX_NUM_COMM_MODES - Maximum number of communication modes
C COMM_NONE - No edge communication
C COMM_MSG - Use messages to communicate edges
C COMM_PUT - Use put to communicate edges
C COMM_GET - Use get to communicate edges
C Note - commName holds an identifying name for each communication
C mode. The COMM_ parameters are used to index commName
C so the COMM_ parameters need to be in the range
C 1 : MAX_NUM_COMM_MODES.
INTEGER MAX_NUM_COMM_MODES
PARAMETER ( MAX_NUM_COMM_MODES = 4 )
INTEGER COMM_NONE
PARAMETER ( COMM_NONE = 1 )
INTEGER COMM_MSG
PARAMETER ( COMM_MSG = 2 )
INTEGER COMM_PUT
PARAMETER ( COMM_PUT = 3 )
INTEGER COMM_GET
PARAMETER ( COMM_GET = 4 )
COMMON /EESUPP_COMMNAME/ commName
CHARACTER*10 commName(MAX_NUM_COMM_MODES)
C Tile identifiers
C Tiles have a number that is unique over the global domain.
C A tile that is not there has its number set to NULL_TILE
INTEGER NULL_TILE
PARAMETER ( NULL_TILE = -1 )
C-- COMMON /EESUPP_C/ Execution environment support character variables
C myProcessStr - String identifying my process number
COMMON /EESUPP_C/ myProcessStr
CHARACTER*128 myProcessStr
C-- COMMON /EESUPP_L/ Execution environment support logical variables
C initMPError - Flag indicating error during multi-processing
C initialisation.
C finMPError - Flag indicating error during multi-processing
C termination.
C ThError - Thread detected an error.
C usingMPI - Flag controlling use of MPI routines. This flag
C allows either MPI or threads to be used in a
C shared memory environment which can be a useful
C debugging/performance analysis tool.
C usingSyncMessages - Flag that causes blocking communication to be used
C if possible. When false non-blocking EXCH routines
C will be used if possible.
C notUsingXPeriodicity - Flag indicating no X/Y boundary wrap around
C notUsingYPeriodicity This affects the communication routines but
C is generally ignored in the numerical model
C code.
C threadIsRunning, threadIsComplete - Flags used to check for correct behaviour
C of multi-threaded code.
C threadIsRunning is used to check that the
C threads we need are running. This catches the
C situation where a program eedata file has nTthreads
C greater than the setenv PARALLEL or NCPUS variable.
C threadIsComplete is used to flag that a thread has
C reached the end of the model. This is used as a check to
C trap problems that might occur if one thread "escapes"
C the main.F master loop. This should not happen
C if the multi-threading compilation tools works right.
C But (see for example KAP) this is not always the case!
COMMON /EESUPP_L/ thError, threadIsRunning, threadIsComplete,
& allMyEdgesAreSharedMemory, usingMPI, usingSyncMessages,
& notUsingXPeriodicity, notUsingYPeriodicity
LOGICAL thError(MAX_NO_THREADS)
LOGICAL threadIsRunning(MAX_NO_THREADS)
LOGICAL threadIsComplete(MAX_NO_THREADS)
LOGICAL allMyEdgesAreSharedMemory(MAX_NO_THREADS)
LOGICAL usingMPI
LOGICAL usingSyncMessages
LOGICAL notUsingXPeriodicity
LOGICAL notUsingYPeriodicity
C-- COMMON /EESUPP_I/ Parallel support integer globals
C pidW - Process ID of neighbor to West
C pidE - ditto East
C pidN - ditto North
C pidS - ditto South
C Note: pid[XY] is not necessairily the UNIX
C process id - it is just an identifying
C number.
C myPid - My own process id
C nProcs - Number of processes
C westCommunicationMode - Mode of communication for each tile face
C eastCommunicationMode
C northCommunicationMode
C southCommunicationMode
C bi0 - Low cartesian tile index for this process
C bj0 Note - In a tile distribution with holes bi0 and bj0
C are not useful. Neighboring tile indices must
C be derived some other way.
C tileNo - Tile identification number for my tile and
C tileNo[WENS] my N,S,E,W neighbor tiles.
C tilePid[WENS] - Process identification number for
C my N,S,E,W neighbor tiles.
C nTx, nTy - No. threads in X and Y. This assumes a simple
C cartesian gridding of the threads which is not
C required elsewhere but that makes it easier.
COMMON /EESUPP_I/
& myPid, nProcs, pidW, pidE, pidN, pidS,
& tileCommModeW, tileCommModeE,
& tileCommModeN, tileCommModeS,
& tileNo, tileNoW, tileNoE, tileNoS, tileNoN,
& tilePidW, tilePidE, tilePidS, tilePidN,
& tileBiW, tileBiE, tileBiS, tileBiN,
& tileBjW, tileBjE, tileBjS, tileBjN,
& tileTagSendW, tileTagSendE, tileTagSendS, tileTagSendN,
& tileTagRecvW, tileTagRecvE, tileTagRecvS, tileTagRecvN
INTEGER myPid
INTEGER nProcs
INTEGER pidW
INTEGER pidE
INTEGER pidN
INTEGER pidS
INTEGER tileCommModeW ( nSx, nSy )
INTEGER tileCommModeE ( nSx, nSy )
INTEGER tileCommModeN ( nSx, nSy )
INTEGER tileCommModeS ( nSx, nSy )
INTEGER tileNo( nSx, nSy )
INTEGER tileNoW( nSx, nSy )
INTEGER tileNoE( nSx, nSy )
INTEGER tileNoN( nSx, nSy )
INTEGER tileNoS( nSx, nSy )
INTEGER tilePidW( nSx, nSy )
INTEGER tilePidE( nSx, nSy )
INTEGER tilePidN( nSx, nSy )
INTEGER tilePidS( nSx, nSy )
INTEGER tileBiW( nSx, nSy )
INTEGER tileBiE( nSx, nSy )
INTEGER tileBiN( nSx, nSy )
INTEGER tileBiS( nSx, nSy )
INTEGER tileBjW( nSx, nSy )
INTEGER tileBjE( nSx, nSy )
INTEGER tileBjN( nSx, nSy )
INTEGER tileBjS( nSx, nSy )
INTEGER tileTagSendW( nSx, nSy )
INTEGER tileTagSendE( nSx, nSy )
INTEGER tileTagSendN( nSx, nSy )
INTEGER tileTagSendS( nSx, nSy )
INTEGER tileTagRecvW( nSx, nSy )
INTEGER tileTagRecvE( nSx, nSy )
INTEGER tileTagRecvN( nSx, nSy )
INTEGER tileTagRecvS( nSx, nSy )
#ifdef ALLOW_USE_MPI
C-- Include MPI standard Fortran header file
#include "mpif.h"
#define _mpiTRUE_ 1
#define _mpiFALSE_ 0
C-- COMMON /EESUPP_MPI_I/ MPI parallel support integer globals
C mpiPidW - MPI process id for west neighbor.
C mpiPidE - MPI process id for east neighbor.
C mpiPidN - MPI process id for north neighbor.
C mpiPidS - MPI process id for south neighbor.
C mpiPidNW - MPI process id for northwest neighbor.
C mpiPidNE - MPI process id for northeast neighbor.
C mpiPidSW - MPI process id for southwest neighbor.
C mpiPidSE - MPI process id for southeast neighbor.
C mpiPidIO - MPI process to use for IO.
C mpiNprocs - No. of MPI processes.
C mpiMyId - MPI process id of me.
C mpiComm - MPI communicator to use.
C mpiPx - My MPI proc. grid X coord
C mpiPy - My MPI proc. grid Y coord
C mpiXGlobalLo - My bottom-left (south-west) x-coordinate in
C global domain.
C mpiYGlobalLo - My bottom-left (south-west) y-coordinate in
C global domain.
C mpiTypeXFaceBlock_xy_r4 - Primitives for communicating edge
C mpiTypeXFaceBlock_xy_r8 of a block.
C mpiTypeYFaceBlock_xy_r4 XFace is used in east-west transfer
C mpiTypeYFaceBlock_xy_r8 YFace is used in nrth-south transfer
C mpiTypeXFaceBlock_xyz_r4 xy is used in two-dimensional arrays
C mpiTypeXFaceBlock_xyz_r8 xyz is used with three-dimensional arrays
C mpiTypeYFaceBlock_xyz_r4 r4 is used for real*4 data
C mpiTypeYFaceBlock_xyz_r8 r8 is used for real*8 data
C mpiTypeXFaceThread_xy_r4 - Composites of the above primitives
C mpiTypeXFaceThread_xy_r8 for communicating edges of all blocks
C mpiTypeYFaceThread_xy_r4 owned by a thread.
C mpiTypeYFaceThread_xy_r8
C mpiTypeXFaceThread_xyz_r4
C mpiTypeXFaceThread_xyz_r8
C mpiTypeYFaceThread_xyz_r4
C mpiTypeYFaceBlock_xyz_r8
C mpiTagE - Tags are needed to mark requests when MPI is running
C mpiTagW between multithreaded processes or when the same process.
C mpiTagS is a neighbor in more than one direction. The tags ensure that
C mpiTagN a thread will get the message it is looking for.
C mpiTagSW The scheme adopted is to tag messages according to
C mpiTagSE the direction they are travelling. Thus a message
C mpiTagNW travelling east is tagged mpiTagE. However, in a
C mpiTagNE multi-threaded environemnt several messages could
C be travelling east from the same process at the
C same time. The tag is therefore modified to
C be mpiTag[EWS...]*nThreads+myThid. This requires that
C each thread also know the thread ids of its "neighbor"
C threads.
COMMON /EESUPP_MPI_I/
& mpiPidW, mpiPidE, mpiPidS, mpiPidN,
& mpiPidSE, mpiPidSW, mpiPidNE, mpiPidNW,
& mpiPidIo, mpiMyId, mpiNProcs, mpiComm,
& mpiPx, mpiPy, mpiXGlobalLo, mpiYGlobalLo,
& mpiTypeXFaceBlock_xy_r4, mpiTypeXFaceBlock_xy_r8,
& mpiTypeYFaceBlock_xy_r4, mpiTypeYFaceBlock_xy_r8,
& mpiTypeXFaceBlock_xyz_r4, mpiTypeXFaceBlock_xyz_r8,
& mpiTypeYFaceBlock_xyz_r4, mpiTypeYFaceBlock_xyz_r8,
& mpiTypeXFaceThread_xy_r4, mpiTypeXFaceThread_xy_r8,
& mpiTypeYFaceThread_xy_r4, mpiTypeYFaceThread_xy_r8,
& mpiTypeXFaceThread_xyz_r4, mpiTypeXFaceThread_xyz_r8,
& mpiTypeYFaceThread_xyz_r4, mpiTypeYFaceThread_xyz_r8,
& mpiTagE, mpiTagW, mpiTagN, mpiTagS,
& mpiTagSE, mpiTagSW, mpiTagNW, mpiTagNE
INTEGER mpiPidW
INTEGER mpiPidE
INTEGER mpiPidS
INTEGER mpiPidN
INTEGER mpiPidSW
INTEGER mpiPidSE
INTEGER mpiPidNW
INTEGER mpiPidNE
INTEGER mpiPidIO
INTEGER mpiMyId
INTEGER mpiNProcs
INTEGER mpiComm
INTEGER mpiPx
INTEGER mpiPy
INTEGER mpiXGlobalLo
INTEGER mpiYGlobalLo
INTEGER mpiTypeXFaceBlock_xy_r4
INTEGER mpiTypeXFaceBlock_xy_r8
INTEGER mpiTypeYFaceBlock_xy_r4
INTEGER mpiTypeYFaceBlock_xy_r8
INTEGER mpiTypeXFaceBlock_xyz_r4
INTEGER mpiTypeXFaceBlock_xyz_r8
INTEGER mpiTypeYFaceBlock_xyz_r4
INTEGER mpiTypeYFaceBlock_xyz_r8
INTEGER mpiTypeXFaceThread_xy_r4(MAX_NO_THREADS)
INTEGER mpiTypeXFaceThread_xy_r8(MAX_NO_THREADS)
INTEGER mpiTypeYFaceThread_xy_r4(MAX_NO_THREADS)
INTEGER mpiTypeYFaceThread_xy_r8(MAX_NO_THREADS)
INTEGER mpiTypeXFaceThread_xyz_r4(MAX_NO_THREADS)
INTEGER mpiTypeXFaceThread_xyz_r8(MAX_NO_THREADS)
INTEGER mpiTypeYFaceThread_xyz_r4(MAX_NO_THREADS)
INTEGER mpiTypeYFaceThread_xyz_r8(MAX_NO_THREADS)
INTEGER mpiTagNW
INTEGER mpiTagNE
INTEGER mpiTagSW
INTEGER mpiTagSE
INTEGER mpiTagW
INTEGER mpiTagE
INTEGER mpiTagN
INTEGER mpiTagS
C-- COMMON /MPI_FULLMAP_I/ holds integer arrays of the full list of MPI process
C mpi_myXGlobalLo :: List of all processors bottom-left X-index in global domain
C mpi_myYGlobalLo :: List of all processors bottom-left Y-index in global domain
C Note: needed for mpi gather/scatter routines & singleCpuIO.
COMMON /MPI_FULLMAP_I/
& mpi_myXGlobalLo, mpi_myYGlobalLo
INTEGER mpi_myXGlobalLo(nPx*nPy)
INTEGER mpi_myYGlobalLo(nPx*nPy)
C MPI communicator describing this model realization
COMMON /MPI_COMMS/
& MPI_COMM_MODEL
INTEGER MPI_COMM_MODEL
#endif /* ALLOW_USE_MPI */