C $Header: /u/gcmpack/MITgcm/eesupp/src/ini_procs.F,v 1.31 2012/09/03 19:29:59 jmc Exp $
C $Name:  $

#include "CPP_EEOPTIONS.h"

CBOP
C     !ROUTINE: INI_PROCS

C     !INTERFACE:
      SUBROUTINE INI_PROCS

C     !DESCRIPTION:
C     *==========================================================*
C     | SUBROUTINE INI\_PROCS
C     | o Initialise multiple concurrent processes environment.
C     *==========================================================*
C     | Under MPI this routine calls various MPI service routines
C     | that map the model grid to MPI processes. The information
C     | is then stored in a common block for later use.
C     | Note: This routine can also be compiled with CPP
C     | directives set so that no multi-processing is initialise.
C     | This is OK and should work fine.
C     *==========================================================*

C     !USES:
      IMPLICIT NONE
C     === Global data ===
#include "SIZE.h"
#include "EEPARAMS.h"
#include "EESUPPORT.h"

#ifdef ALLOW_USE_MPI
C     !FUNCTIONS:

C     !LOCAL VARIABLES:
C     === Local variables ===
C     msgBuf         :: IO buffer
C     myThid         :: Dummy thread id
C     mpiRC          :: Error code reporting variable used with MPI.
C     mpiGridSpec    :: No. of processes in X and Y.
C     mpiPeriodicity :: Flag indicating XY priodicity to MPI.
C     arrElSize      :: Size of an array element in bytes used to define
C                       MPI datatypes for communication operations.
C     arrElSep       :: Separation in units of array elements between
C                       blocks to be communicated.
C     elCount        :: No. of blocks that are associated with MPI datatype.
C     elLen          :: Length of an MPI datatype in terms of preexisting
C                       datatype.
C     elStride       :: Distance between starting location of elements in
C                       an MPI datatype - can be bytes of datatype units.
      INTEGER mpiRC
      INTEGER mpiGridSpec(2)
      INTEGER mpiPeriodicity(2)
      INTEGER mpiLProcNam
      CHARACTER*(MPI_MAX_PROCESSOR_NAME) mpiProcNam
      INTEGER arrElSize
      INTEGER arrElSep
      INTEGER elCount
      INTEGER elLen
      INTEGER elStride
      INTEGER np, pId, itemp(2)
      INTEGER ierr
#endif /* ALLOW_USE_MPI */
      CHARACTER*(MAX_LEN_MBUF) msgBuf
      INTEGER myThid
CEOP

C--   Default values set to single processor case
C     pid[W-SE] are the MPI process id of the neighbor processes.
C     A process can be its own neighbor!
      myThid      = 1
      myPid       = 0
      nProcs      = 1
      myPx        = 1
      myPy        = 1
      myXGlobalLo = 1
      myYGlobalLo = 1
      pidW        = 0
      pidE        = 0
      pidN        = 0
      pidS        = 0
c     errorMessageUnit    = 0
c     standardMessageUnit = 6

      IF ( usingMPI ) THEN
#ifdef ALLOW_USE_MPI
C--
C--   MPI style full multiple-process initialisation
C--   ==============================================

C--    Arrange MPI processes on a cartesian grid
C      Set variable indicating which MPI process is to the north,
C      south, east, west, south-west, south-east, north-west
C      and north-east of me e.g.
C
C      Plan view of model domain centered on process ME
C      ================================================
C
C            :         :         :        :
C            :         :         :        :
C            :         :         :        :
C       .....------------------------------.....
C            |         |         |        |
C            |  NW     |   N     |  NE    |
C            |         |         |        |
C       .....------------------------------.....
C            |         |         |        |
C            |  W      |   ME    |  E     |
C            |         |         |        |
C       .....------------------------------.....
C            |         |         |        |
C            |  SW     |   S     |  SE    |
C            |         |         |        |
C       .....------------------------------.....
C  Y         :         :         :        :
C / \        :         :         :        :
C  |         :         :         :        :
C  |
C  |----> X
C
C--    Set default MPI communicator to XY processor grid
       mpiGridSpec(1) = nPx
       mpiGridSpec(2) = nPy
C      Could be periodic in X and/or Y - set at run time or compile time!
       mpiPeriodicity(1) = _mpiTRUE_
       mpiPeriodicity(2) = _mpiTRUE_
#ifdef CAN_PREVENT_X_PERIODICITY
#ifndef ALWAYS_PREVENT_X_PERIODICITY
       IF ( notUsingXPeriodicity ) THEN
#endif
        mpiPeriodicity(1) = _mpiFALSE_
#ifndef ALWAYS_PREVENT_X_PERIODICITY
       ENDIF
#endif
#endif /* CAN_PREVENT_X_PERIODICITY */
#ifdef  CAN_PREVENT_Y_PERIODICITY
#ifndef ALWAYS_PREVENT_Y_PERIODICITY
       IF ( notUsingYPeriodicity ) THEN
#endif
        mpiPeriodicity(2) = _mpiFALSE_
#ifndef ALWAYS_PREVENT_Y_PERIODICITY
       ENDIF
#endif
#endif /* CAN_PREVENT_Y_PERIODICITY */

       CALL MPI_CART_CREATE(
     I  MPI_COMM_MODEL,2,mpiGridSpec,mpiPeriodicity,_mpiTRUE_,
     O  mpiComm, mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_CART_CREATE return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF

C--    Get my location on the grid
       CALL MPI_CART_COORDS( mpiComm, mpiMyId, 2, mpiGridSpec, mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_CART_COORDS return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF
       myPid = mpiMyId
       mpiPx = mpiGridSpec(1)
       mpiPy = mpiGridSpec(2)
       mpiXGlobalLo = 1 + sNx*nSx*(mpiPx)
       mpiYGlobalLo = 1 + sNy*nSy*(mpiPy)
       myXGlobalLo  = mpiXGlobalLo
       myYGlobalLo  = mpiYGlobalLo

C--   To speed-up mpi gather and scatter routines, myXGlobalLo
C     and myYGlobalLo from each process are transferred to
C     a common block array.  This allows process 0 to know
C     the location of the domains controlled by each process.
       DO np = 1, nPx*nPy
          itemp(1) = myXGlobalLo
          itemp(2) = myYGlobalLo
          pId = np - 1
          CALL MPI_BCAST(itemp, 2, MPI_INTEGER, pId,
     &         MPI_COMM_MODEL, ierr)
          mpi_myXGlobalLo(np) = itemp(1)
          mpi_myYGlobalLo(np) = itemp(2)
       ENDDO

       myPx = mpiPx+1
       myPy = mpiPy+1
C--    Get MPI id for neighboring procs.
       mpiGridSpec(1) = mpiPx-1
       IF ( mpiPeriodicity(1) .EQ. _mpiTRUE_
     &   .AND. mpiGridSpec(1) .LT. 0 )
     &  mpiGridSpec(1) = nPx-1
       mpiGridSpec(2) = mpiPy

#ifdef ALLOW_NEST_CHILD
      IF ( useNEST_CHILD) THEN
       IF ( mpiPeriodicity(1) .EQ. _mpiFALSE_
     &      .AND. mpiGridSpec(1) .LT. 0 )
     &      mpiGridSpec(1) =  0
      ENDIF
#endif /* ALLOW_NEST_CHILD */

       CALL MPI_CART_RANK( mpiComm, mpiGridSpec, mpiPidW , mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_CART_RANK (pidW) return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF
       pidW = mpiPidW
       mpiGridSpec(1) = mpiPx+1
       IF ( mpiPeriodicity(1) .EQ. _mpiTRUE_
     &   .AND. mpiGridSpec(1) .GT. nPx-1 )
     &  mpiGridSpec(1) = 0
       mpiGridSpec(2) = mpiPy

#ifdef ALLOW_NEST_CHILD
      IF ( useNEST_CHILD) THEN
       IF ( mpiPeriodicity(1) .EQ. _mpiFALSE_
     &   .AND. mpiGridSpec(1) .GT. nPx-1 )
     &    mpiGridSpec(1) = nPx-1
      ENDIF
#endif /* ALLOW_NEST_CHILD */

       CALL MPI_CART_RANK( mpiComm, mpiGridSpec, mpiPidE , mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_CART_RANK (pidE) return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF
       pidE = mpiPidE
       mpiGridSpec(1) = mpiPx
       mpiGridSpec(2) = mpiPy-1
       IF ( mpiPeriodicity(2) .EQ. _mpiTRUE_
     &   .AND. mpiGridSpec(2) .LT. 0 )
     &  mpiGridSpec(2) = nPy - 1
       CALL MPI_CART_RANK( mpiComm, mpiGridSpec, mpiPidS , mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_CART_RANK (pidS) return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF
       pidS = mpiPidS
       mpiGridSpec(1) = mpiPx
       mpiGridSpec(2) = mpiPy+1
       IF ( mpiPeriodicity(2) .EQ. _mpiTRUE_
     &   .AND. mpiGridSpec(2) .GT. nPy-1 )
     &  mpiGridSpec(2) = 0
       CALL MPI_CART_RANK( mpiComm, mpiGridSpec, mpiPidN , mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_CART_RANK (pidN) return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF
       pidN = mpiPidN

C--    Print summary of processor mapping on standard output
       CALL MPI_GET_PROCESSOR_NAME( mpiProcNam, mpilProcNam, mpiRC )
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &        'S/R INI_PROCS: MPI_GET_PROCESSOR_NAME return code',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF
       WRITE(msgBuf,'(A)')
     &   '======= Starting MPI parallel Run ========='
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_BOTH , myThid )
       WRITE(msgBuf,'(A,I3,A,A)') ' My Processor Name (len:',
     &  mpilProcNam, ' ) = ', mpiProcNam(1:mpilProcNam)
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )
       WRITE(msgBuf,'(A,I3,A,I3,A,I3,A,I3,A)') ' Located at (',
     &  mpiPx,',',mpiPy,
     &  ') on processor grid (0:',nPx-1,',0:',nPy-1,')'
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )
       WRITE(msgBuf,'(A,I6,A,I6,A,I6,A,I6,A)') ' Origin at  (',
     &  mpiXGlobalLo,',',mpiYGLobalLo,
     &  ') on global grid (1:',nPx*sNx*nSx,',1:',nPy*sNy*nSy,')'
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )
       WRITE(msgBuf,'(A,I4.4)')
     &   ' North neighbor = processor ', mpiPidN
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )
       WRITE(msgBuf,'(A,I4.4)')
     &   ' South neighbor = processor ', mpiPidS
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )
       WRITE(msgBuf,'(A,I4.4)')
     &   '  East neighbor = processor ', mpiPidE
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )
       WRITE(msgBuf,'(A,I4.4)')
     &   '  West neighbor = processor ', mpiPidW
       CALL PRINT_MESSAGE( msgBuf, standardMessageUnit,
     &                     SQUEEZE_RIGHT , myThid )

C--    Create MPI types for transfer of array edges.
C--    Four and eight byte primitive (one block only) datatypes.
C--    These are common to all threads in the process.
C      Notes:
C      ======
C      1. The datatypes MPI_REAL4 and MPI_REAL8 are usually predefined.
C      If they are not defined code must be added to create them -
C      the MPI standard leaves optional whether they exist.
C      2. Per thread datatypes that handle all the edges for a thread
C      are defined based on the type defined here.

C--    xFace datatypes (east<-->west messages)
C--
C      xFace (y=constant) for XY arrays with real*4 declaration.
       arrElSep  = (sNx+OLx*2)
       elCount   = sNy+OLy*2
       elLen     = OLx
       elStride  = arrElSep
#if (defined (TARGET_SGI)  defined (TARGET_AIX)  defined(TARGET_LAM))
       CALL MPI_TYPE_VECTOR(elCount,elLen,elStride,MPI_REAL,
     &                       mpiTypeXFaceBlock_xy_r4, mpiRC)
#else
       CALL MPI_TYPE_VECTOR(elCount,elLen,elStride,MPI_REAL4,
     &                       mpiTypeXFaceBlock_xy_r4, mpiRC)
#endif
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_VECTOR (mpiTypeXFaceBlock_xy_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeXFaceBlock_xy_r4, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeXFaceBlock_xy_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF

C      xFace (y=constant) for XY arrays with real*8 declaration.
#if (defined (TARGET_SGI)  defined (TARGET_AIX)  defined(TARGET_LAM))
       CALL MPI_TYPE_VECTOR(elCount,elLen,elStride,MPI_DOUBLE_PRECISION,
     &                       mpiTypeXFaceBlock_xy_r8, mpiRC)
#else
       CALL MPI_TYPE_VECTOR(elCount,elLen,elStride,MPI_REAL8,
     &                       mpiTypeXFaceBlock_xy_r8, mpiRC)
#endif
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_VECTOR (mpiTypeXFaceBlock_xy_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeXFaceBlock_xy_r8, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeXFaceBlock_xy_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF

C      xFace (y=constant) for XYZ arrays with real*4 declaration.
       arrElSize = 4
       arrElSep  = (sNx+OLx*2)*(sNy+OLy*2)
       elCount   = Nr
       elLen     = 1
       elStride  = arrElSize*arrElSep
       CALL MPI_TYPE_HVECTOR(elCount,elLen,elStride,
     &                        mpiTypeXFaceBlock_xy_r4,
     &                       mpiTypeXFaceBlock_xyz_r4, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_HVECTOR (mpiTypeXFaceBlock_xyz_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeXFaceBlock_xyz_r4, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT  (mpiTypeXFaceBlock_xyz_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF

C      xFace (y=constant) for XYZ arrays with real*8 declaration.
       arrElSize = 8
       elStride  = arrElSize*arrElSep
       CALL MPI_TYPE_HVECTOR(elCount,elLen,elStride,
     &                        mpiTypeXFaceBlock_xy_r8,
     &                       mpiTypeXFaceBlock_xyz_r8, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_HVECTOR (mpiTypeXFaceBlock_xyz_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeXFaceBlock_xyz_r8, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeXFaceBlock_xyz_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF

C--    yFace datatypes (north<-->south messages)
C--
C      yFace (x=constant) for XY arrays with real*4 declaration
       elCount  = OLy*(sNx+OLx*2)
#if (defined (TARGET_SGI)  defined (TARGET_AIX)  defined(TARGET_LAM))
       CALL MPI_TYPE_CONTIGUOUS(elCount,MPI_REAL,
     &                          mpiTypeYFaceBlock_xy_r4, mpiRC)
#else
       CALL MPI_TYPE_CONTIGUOUS(elCount,MPI_REAL4,
     &                          mpiTypeYFaceBlock_xy_r4, mpiRC)
#endif
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_CONTIGUOUS (mpiTypeYFaceBlock_xy_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeYFaceBlock_xy_r4, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeYFaceBlock_xy_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
C      yFace (x=constant) for XY arrays with real*8 declaration
#if (defined (TARGET_SGI)  defined (TARGET_AIX)  defined(TARGET_LAM))
       CALL MPI_TYPE_CONTIGUOUS(elCount,MPI_DOUBLE_PRECISION,
     &                          mpiTypeYFaceBlock_xy_r8, mpiRC)
#else
       CALL MPI_TYPE_CONTIGUOUS(elCount,MPI_REAL8,
     &                          mpiTypeYFaceBlock_xy_r8, mpiRC)
#endif
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_CONTIGUOUS (mpiTypeYFaceBlock_xy_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeYFaceBlock_xy_r8, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeYFaceBlock_xy_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
C      yFace (x=constant) for XYZ arrays with real*4 declaration
       arrElSize = 4
       arrElSep  = (sNx+OLx*2)*(sNy+OLy*2)
       elCount   = Nr
       elLen     = 1
       elStride  = arrElSize*arrElSep
       CALL MPI_TYPE_HVECTOR(elCount,elLen,elStride,
     &                        mpiTypeYFaceBlock_xy_r4,
     &                       mpiTypeYFaceBlock_xyz_r4, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_HVECTOR (mpiTypeYFaceBlock_xyz_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeYFaceBlock_xyz_r4, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeYFaceBlock_xyz_r4)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
C      yFace (x=constant) for XYZ arrays with real*8 declaration
       arrElSize = 8
       elStride  = arrElSize*arrElSep
       CALL MPI_TYPE_HVECTOR(elCount,elLen,elStride,
     &                        mpiTypeYFaceBlock_xy_r8,
     &                       mpiTypeYFaceBlock_xyz_r8, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_HVECTOR (mpiTypeYFaceBlock_xyz_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF
       CALL MPI_TYPE_COMMIT( mpiTypeYFaceBlock_xyz_r8, mpiRC)
       IF ( mpiRC .NE. MPI_SUCCESS ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(A,I5)')
     &   'S/R INI_PROCS: MPI_TYPE_COMMIT (mpiTypeYFaceBlock_xyz_r8)',
     &        mpiRC
        CALL PRINT_ERROR( msgBuf, myThid )
       ENDIF

C--    Assign MPI values used in generating unique tags for messages.
       mpiTagW    = 1
       mpiTagE    = 2
       mpiTagS    = 3
       mpiTagN    = 4

       CALL MPI_BARRIER(MPI_COMM_MODEL,mpiRC)

#endif /* ALLOW_USE_MPI */
      ELSE
C--   Case without using MPI (usingMPI=F)

C--   case without tile-communication (DISCONNECTED_TILES defined) is not
C     yet coded for multi-procs; for now, just stop if multi-procs domain
       IF ( nPx*nPy .NE. 1 ) THEN
        eeBootError = .TRUE.
        WRITE(msgBuf,'(2A,I6,A)') 'INI_PROCS: ',
     &    'needs MPI for multi-procs (nPx*nPy=',  nPx*nPy, ') setup'
        CALL PRINT_ERROR( msgBuf, myThid )
        WRITE(msgBuf,'(2A)') 'INI_PROCS: ',
     &    ' but presently usingMPI = False (in "eedata")'
        CALL PRINT_ERROR( msgBuf, myThid )
        GOTO 999
       ENDIF

C--   End if usingMPI
      ENDIF

 999  CONTINUE

      RETURN
      END