/orcd/pool/004/jahn/centos7/MITgcm_gfortran-mpi-fast/verification/global_ocean.90x40x15/tr_run.dwnslp
../../../tools/tst_2+2 -mpi All -command "mpirun -n 5 -hostfile /tmp/machinefile.54695 ./mitgcmuv"
 
cmdEXE='mpirun -n 5 -hostfile /tmp/machinefile.54695 ./mitgcmuv'
from previous run STDOUT.0000, lastPick='    36010 ckptA' ; iter='36010' ; sufx='ckptA'
prepare parameter file 'data.tst' :
prepare file 'data.tst' : done
 diff data.tst data
48,52c48,50
<  nIter0=36010,
<  nTimeSteps=4,
< # pickupStrictlyMatch=.FALSE.,
< # nIter0=36000,
< # nTimeSteps=10,
---
>  pickupStrictlyMatch=.FALSE.,
>  nIter0=36000,
>  nTimeSteps=10,
62,63c60,61
< # pChkPtFreq=1555200000.,
< # chkptFreq=  311040000.,
---
>  pChkPtFreq=1555200000.,
>  chkptFreq=  311040000.,
 
 link back: temp_tst/pickup*.ckptA*
 rename ckptA -> 0000036010 for all: pickup_cd pickup pickup_ptracers
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036010
  rnp_loc: pickup.ckptA pickup.0000036010
  rnp_loc: pickup_ptracers.ckptA pickup_ptracers.0000036010
 start-end iter: 36010 , 36012 , 36014
 sufix: '0000036010' '0000036012' '0000036014'
 cmdEXE=mpirun -n 5 -hostfile /tmp/machinefile.54695 ./mitgcmuv
==> START RUN 2 x 2 it
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
==>  END  RUN 2 x 2 it
 listP= pickup_cd pickup pickup_ptracers
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036014
  rnp_loc: pickup.ckptA pickup.0000036014
  rnp_loc: pickup_ptracers.ckptA pickup_ptracers.0000036014
 move_outp: res_2it
==> START RUN 1iA
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
==>  END  RUN 1iA
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036012
  rnp_loc: pickup.ckptA pickup.0000036012
  rnp_loc: pickup_ptracers.ckptA pickup_ptracers.0000036012
 move_outp: res_1iA
==> START RUN 1iB
STOP ABNORMAL END: S/R MDS_READ_FIELD
STOP ABNORMAL END: S/R MDS_READ_FIELD
[node669:mpi_rank_2][handle_cqe] Send desc error in msg to 3, wc_opcode=0
[node669:mpi_rank_2][handle_cqe] Msg from 3: wc.status=12, wc.wr_id=0x3651550, wc.opcode=0, vbuf->phead->type=0 = MPIDI_CH3_PKT_EAGER_SEND
[node669:mpi_rank_2][handle_cqe] src/mpid/ch3/channels/mrail/src/gen2/ibv_channel_manager.c:548: [] Got completion with error 12, vendor code=0x81, dest rank=3
: No such file or directory (2)
[node669:mpi_rank_1][handle_cqe] Send desc error in msg to 3, wc_opcode=0
[node669:mpi_rank_1][handle_cqe] Msg from 3: wc.status=12, wc.wr_id=0x36522d0, wc.opcode=0, vbuf->phead->type=0 = MPIDI_CH3_PKT_EAGER_SEND
[node669:mpi_rank_1][handle_cqe] src/mpid/ch3/channels/mrail/src/gen2/ibv_channel_manager.c:548: [] Got completion with error 12, vendor code=0x81, dest rank=3
: No such file or directory (2)
==> RUN 1iB STOP without writing pickup => exit