/home/jm_c/test_engaging/MITgcm_ifcMpi/verification/cfc_example/run
../../../tools/tst_2+2 -mpi All -command "mpirun -np 2 ./mitgcmuv"
 
cmdEXE='mpirun -np 2 ./mitgcmuv'
from previous run STDOUT.0000, lastPick='  4269604 ckptA' ; iter='4269604' ; sufx='ckptA'
prepare parameter file 'data.tst' :
prepare file 'data.tst' : done
 diff data.tst data
37,40c37,38
<  nIter0=4269604,
<  nTimeSteps=4,
< # nIter0 = 4269600,
< # nTimeSteps = 4,
---
>  nIter0 = 4269600,
>  nTimeSteps = 4,
51,52c49,50
< # pChkptFreq=3110400000.,
< # chkptFreq= 3110400000.,
---
>  pChkptFreq=3110400000.,
>  chkptFreq= 3110400000.,
 
 link back: temp_tst/pickup*.ckptA*
 rename ckptA -> 0004269604 for all: pickup_cd pickup pickup_ptracers
  rnp_loc: pickup_cd.ckptA pickup_cd.0004269604
  rnp_loc: pickup.ckptA pickup.0004269604
  rnp_loc: pickup_ptracers.ckptA pickup_ptracers.0004269604
 start-end iter: 4269604 , 4269606 , 4269608
 sufix: '0004269604' '0004269606' '0004269608'
 cmdEXE=mpirun -np 2 ./mitgcmuv
==> START RUN 2 x 2 it
 QQ start cfc params
 QQ read cfc atmos data
 QQ start cfc params
 QQ read cfc atmos data
NORMAL END
NORMAL END
==>  END  RUN 2 x 2 it
 listP= pickup_cd pickup pickup_ptracers
  rnp_loc: pickup_cd.ckptA pickup_cd.0004269608
  rnp_loc: pickup.ckptA pickup.0004269608
  rnp_loc: pickup_ptracers.ckptA pickup_ptracers.0004269608
 move_outp: res_2it
==> START RUN 1iA
mv: cannot stat 'STDOUT.0000': No such file or directory
[mpiexec@node3303] Error: Unable to run bstrap_proxy on node3303 (pid 2930402, exit code 256)
[mpiexec@node3303] poll_for_event (../../../../../src/pm/i_hydra/libhydra/demux/hydra_demux_poll.c:157): check exit codes error
[mpiexec@node3303] HYD_dmx_poll_wait_for_proxy_event (../../../../../src/pm/i_hydra/libhydra/demux/hydra_demux_poll.c:206): poll for event error
[mpiexec@node3303] HYD_bstrap_setup (../../../../../src/pm/i_hydra/libhydra/bstrap/src/intel/i_hydra_bstrap.c:1069): error waiting for event
[mpiexec@node3303] Error setting up the bootstrap proxies
[mpiexec@node3303] Possible reasons:
[mpiexec@node3303] 1. Host is unavailable. Please check that all hosts are available.
[mpiexec@node3303] 2. Cannot launch hydra_bstrap_proxy or it crashed on one of the hosts.
[mpiexec@node3303]    Make sure hydra_bstrap_proxy is available on all hosts and it has right permissions.
[mpiexec@node3303] 3. Firewall refused connection.
[mpiexec@node3303]    Check that enough ports are allowed in the firewall and specify them with the I_MPI_PORT_RANGE variable.
[mpiexec@node3303] 4. slurm bootstrap cannot launch processes on remote host.
[mpiexec@node3303]    You may try using -bootstrap option to select alternative launcher.
srun: error: Unable to create step for job 22440792: Memory required by task is not available
empty or no output file: 'std_outp.1iA' => exit