C $Header: /u/gcmpack/MITgcm/eesupp/src/ini_communication_patterns.F,v 1.11 2012/09/01 17:37:30 jmc Exp $ C $Name: $ #include "PACKAGES_CONFIG.h" #include "CPP_EEOPTIONS.h" CBOP C !ROUTINE: INI_COMMUNICATION_PATTERNS C !INTERFACE: SUBROUTINE INI_COMMUNICATION_PATTERNS( myThid ) C !DESCRIPTION: C *==========================================================* C | SUBROUTINE INI\_COMMUNICATION\_PATTERNS C | o Initialise between tile communication data structures. C *==========================================================* C | This routine assigns identifiers to each tile and then C | defines a map of neighbors for each tile. C | For each neighbor a communication method is defined. C *==========================================================* C !USES: IMPLICIT NONE C === Global data === #include "SIZE.h" #include "EEPARAMS.h" #include "EESUPPORT.h" #include "EXCH.h" C !INPUT/OUTPUT PARAMETERS: C === Routine arguments === C myThid :: Thread number we are dealing with in this call INTEGER myThid C !LOCAL VARIABLES: C === Local variables === C pxW :: Process X coord of process to west. C pxE :: Process X coord of process to west. C pyN :: Process Y coord of process to north. C pyS :: Process Y coord of process to south. C procW :: Process Id of process to west. C procE :: Process Id of process to east. C procN :: Process Id of process to north. C procS :: Process Id of process to south. C totalTileCount :: Total number of tiles C tagW0, tagE0, tagS0, tagN0, theTag :: Working variables for C calculating message tags. C biW, biE, bjN, bjS :: Tile x and y indices to west, east, C south and north. C bi, bj :: Tile loop counter C picnt, pjcnt :: Process loop counter C bi0, bj0 :: Base global index coordinate ( on CS there is no global C coord). INTEGER bi0(nPx) INTEGER bj0(nPy) INTEGER bi, bj, picnt, pjcnt INTEGER pxW, pxE, pyN, pyS INTEGER procW, procE, procN, procS INTEGER totalTileCount INTEGER tagW0, tagE0, tagS0, tagN0, theTag INTEGER biE, biW, bjN, bjS INTEGER thePx, thePy, theBj, theBi CEOP C-- Define a globally unique tile numbers for each tile. C-- We aslo define the tile numbers for our east, west, south C-- and north neighbor tiles here. As coded below this is done from C-- a simple cartesian formula. To handle irregular tile distributions C-- the code below would be changed. For instance we could read C-- the neighbor tile information from a file rather than deriving C-- it in-line. This allows general tile distributions and connectivity C-- both within a thread, between threads and between processors. C Notes -- C 1. The cartesian based formula coded below works as follows: C i. Each tile has one west neighbor, one east neighbor C one north neignbor and one south neighbor. C ii. For each of my neighbors store the following C - neighbor tile id C - neighbor process id C 2. The information that is stored is then used to determine C the between tile communication method. The method used C depends on whether the tile is part of the same process, C on the same machine etc... C 3. To initialise a tile distribution with holes in it C i.e. tiles that are not computed on. Set tile number to C the value NULL_TILE. This must also be done for tileNoW, C tileNoE, tileNoS, tileNoN. C 4. The default formula below assigns tile numbers sequentially C in X on the **global** grid. Within a process the tile numbers C will not necessairily be sequential. This means that the tile C numbering label does not change when nTx, nTy, nPx or nPy change. C It will only change if the tile size changes or the global C grid changes. C bi0 and bj0 are the base global tile grid coordinate for the first C tile in this process. DO picnt = 1, nPx bi0(picnt) = picnt ENDDO DO pjcnt = 1, nPy bj0(pjcnt) = pjcnt ENDDO DO bj=myByLo(myThid),myByHi(myThid) DO bi=myBxLo(myThid),myBxHi(myThid) C o My tile identifier Crg tileNo(bi,bj) = (bj0(myPy)-1+bj-1)*nSx*nPx+bi0(myPx)+bi-1 thePx = myPx thePy = myPy theBj = bj theBi = bi tileNo(bi,bj) = & ((thePy-1)*nSy+theBj-1)*nSx*nPx & + (thePx-1)*nSx & + theBi C o My west neighbor tile and process identifier biW = bi-1 pxW = myPx procW = myPid IF ( biW .LT. 1 ) THEN biW = nSx pxW = myPx-1 procW = pidW IF ( pxW .LT. 1 ) pxW = nPx ENDIF Crg tileNoW (bi,bj) = (bj0(myPy)-1+bj-1)*nSx*nPx+bi0(pxW)+biW-1 thePx = pxW thePy = myPy theBj = bj theBi = biW tileNoW (bi,bj) = & ((thePy-1)*nSy+theBj-1)*nSx*nPx & + (thePx-1)*nSx & + theBi #ifdef ALLOW_NEST_CHILD #ifndef ALLOW_USE_MPI tileNoW (bi,bj) = NULL_TILE #endif #endif tilePidW(bi,bj) = procW tileBiW (bi,bj) = biW tileBjW (bi,bj) = bj C o My east neighbor tile and process identifier biE = bi+1 pxE = myPx procE = myPid IF ( biE .GT. nSx ) THEN biE = 1 pxE = myPx+1 procE = pidE IF ( pxE .GT. nPx ) pxE = 1 ENDIF Crg tileNoE(bi,bj) = (bj0(myPy)-1+bj-1)*nSx*nPx+bi0(pxE)+biE-1 thePx = pxE thePy = myPy theBi = biE theBj = bj tileNoE(bi,bj) = & ((thePy-1)*nSy+theBj-1)*nSx*nPx & + (thePx-1)*nSx & + theBi #ifdef ALLOW_NEST_CHILD #ifndef ALLOW_USE_MPI tileNoE (bi,bj) = NULL_TILE #endif #endif tilePidE(bi,bj) = procE tileBiE (bi,bj) = biE tileBjE (bi,bj) = bj C o My north neighbor tile and process identifier bjN = bj+1 pyN = myPy procN = myPid IF ( bjN .GT. nSy ) THEN bjN = 1 pyN = myPy+1 procN = pidN IF ( pyN .GT. nPy ) pyN = 1 ENDIF Crg tileNoN(bi,bj) = (bj0(pyN)-1+bjN-1)*nSx*nPx+bi0(myPx)+bi-1 thePx = myPx thePy = pyN theBi = bi theBj = bjN tileNoN(bi,bj) = & ((thePy-1)*nSy+theBj-1)*nSx*nPx & + (thePx-1)*nSx & + theBi tilePidN(bi,bj) = procN tileBiN(bi,bj) = bi tileBjN(bi,bj) = bjN C o My south neighbor tile and process identifier bjS = bj-1 pyS = myPy procS = myPid IF ( bjS .LT. 1 ) THEN bjS = nSy pyS = pyS-1 procS = pidS IF ( pyS .LT. 1 ) pyS = nPy ENDIF Crg tileNoS(bi,bj) = (bj0(pyS+1)-1+bjS-1)*nSx*nPx+bi0(myPx+1)+bi-1 thePx = myPx thePy = pyS theBi = bi theBj = bjS tileNoS(bi,bj) = & ((thePy-1)*nSy+theBj-1)*nSx*nPx & + (thePx-1)*nSx & + theBi tilePidS(bi,bj) = procS tileBiS(bi,bj) = bi tileBjS(bi,bj) = bjS ENDDO ENDDO C-- Define the total count of tiles. totalTileCount = nSx*nSy*nPx*nPy C-- Set tags for each tile face. C Tags are used to distinguish exchanges from particular C faces of particular tiles. C Tag numbers are based on C i - The tile number C ii - The direction (N,S,W,E) of the message C We dont check for the NULL_TILE tile number here as it C should not actually be used. TagW0=1 TagE0=2 TagN0=3 TagS0=4 DO bj=myByLo(myThid),myByHi(myThid) DO bi=myBxLo(myThid),myBxHi(myThid) C Send tags C o Tag I use for messages I send to west theTag = TagW0*totalTileCount+tileNo(bi,bj)-1 tileTagSendW(bi,bj) = theTag C o Tag I use for messages I send to east theTag = TagE0*totalTileCount+tileNo(bi,bj)-1 tileTagSendE(bi,bj) = theTag C o Tag I use for messages I send to north theTag = TagN0*totalTileCount+tileNo(bi,bj)-1 tileTagSendN(bi,bj) = theTag C o Tag I use for messages I send to south theTag = TagS0*totalTileCount+tileNo(bi,bj)-1 tileTagSendS(bi,bj) = theTag C Receive tags C o Tag on messages I receive from my east theTag = TagW0*totalTileCount+tileNoE(bi,bj)-1 tileTagRecvE(bi,bj) = theTag C o Tag on messages I receive from my west theTag = TagE0*totalTileCount+tileNoW(bi,bj)-1 tileTagRecvW(bi,bj) = theTag C o Tag on messages I receive from my north theTag = TagS0*totalTileCount+tileNoN(bi,bj)-1 tileTagRecvN(bi,bj) = theTag C o Tag on messages I receive from my north theTag = TagN0*totalTileCount+tileNoS(bi,bj)-1 tileTagRecvS(bi,bj) = theTag ENDDO ENDDO C-- Set the form of excahnge to use between neighboring tiles. C For now use either shared memory, messages or nothing. Further C rules can be added later to allow shm regions and ump regions etc... C Notes - C 1. We require symmetry here. If one face of a tile uses C communication method A then the matching face on its neighbor C tile must also use communication method A. DO bj=myByLo(myThid),myByHi(myThid) DO bi=myBxLo(myThid),myBxHi(myThid) C o West face communication IF ( tileNoW(bi,bj) .EQ. NULL_TILE ) THEN tileCommModeW(bi,bj) = COMM_NONE ELSE IF ( myPid .EQ. tilePidW(bi,bj) ) THEN tileCommModeW(bi,bj) = COMM_PUT ELSE tileCommModeW(bi,bj) = COMM_MSG ENDIF ENDIF C o East face communication IF ( tileNoE(bi,bj) .EQ. NULL_TILE ) THEN tileCommModeE(bi,bj) = COMM_NONE ELSE IF ( myPid .EQ. tilePidE(bi,bj) ) THEN tileCommModeE(bi,bj) = COMM_PUT ELSE tileCommModeE(bi,bj) = COMM_MSG ENDIF ENDIF C o South face communication IF ( tileNoS(bi,bj) .EQ. NULL_TILE ) THEN tileCommModeS(bi,bj) = COMM_NONE ELSE IF ( myPid .EQ. tilePidS(bi,bj) ) THEN tileCommModeS(bi,bj) = COMM_PUT ELSE tileCommModeS(bi,bj) = COMM_MSG ENDIF ENDIF C o North face communication IF ( tileNoN(bi,bj) .EQ. NULL_TILE ) THEN tileCommModeN(bi,bj) = COMM_NONE ELSE IF ( myPid .EQ. tilePidN(bi,bj) ) THEN tileCommModeN(bi,bj) = COMM_PUT ELSE tileCommModeN(bi,bj) = COMM_MSG ENDIF ENDIF ENDDO ENDDO C Initialise outstanding exchange request counter DO bj=myByLo(myThid),myByHi(myThid) DO bi=myBxLo(myThid),myBxHi(myThid) exchNReqsX(1,bi,bj) = 0 exchNReqsY(1,bi,bj) = 0 ENDDO ENDDO RETURN END