/orcd/pool/004/jahn/centos7/MITgcm_gfortran-mpi/verification/aim.5l_cs/run
../../../tools/tst_2+2 -mpi All -command "mpirun -n 6 -hostfile /tmp/machinefile.54697 ./mitgcmuv"
 
cmdEXE='mpirun -n 6 -hostfile /tmp/machinefile.54697 ./mitgcmuv'
from previous run STDOUT.0000, lastPick='    69130 ckptA' ; iter='69130' ; sufx='ckptA'
prepare parameter file 'data.tst' :
prepare file 'data.tst' : done
 diff data.tst data
43,46c43,44
<  nIter0=69130,
<  nTimeSteps=4,
< # nIter0=69120,
< # nTimeSteps=10,
---
>  nIter0=69120,
>  nTimeSteps=10,
54c52
< # pChkptFreq=2592000.,
---
>  pChkptFreq=2592000.,
 
 link back: temp_tst/pickup*.ckptA*
 rename ckptA -> 0000069130 for all: pickup pickup_land
  rnp_loc: pickup.ckptA pickup.0000069130
  rnp_loc: pickup_land.ckptA pickup_land.0000069130
 start-end iter: 69130 , 69132 , 69134
 sufix: '0000069130' '0000069132' '0000069134'
 cmdEXE=mpirun -n 6 -hostfile /tmp/machinefile.54697 ./mitgcmuv
==> START RUN 2 x 2 it
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
Note: The following floating-point exceptions are signalling: IEEE_UNDERFLOW_FLAG
STOP NORMAL END
STOP NORMAL END
==>  END  RUN 2 x 2 it
 listP= pickup pickup_land
  rnp_loc: pickup.ckptA pickup.0000069134
  rnp_loc: pickup_land.ckptA pickup_land.0000069134
 move_outp: res_2it
==> START RUN 1iA
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
Note: The following floating-point exceptions are signalling: IEEE_UNDERFLOW_FLAG
STOP NORMAL END
STOP NORMAL END
==>  END  RUN 1iA
  rnp_loc: pickup.ckptA pickup.0000069132
  rnp_loc: pickup_land.ckptA pickup_land.0000069132
 move_outp: res_1iA
==> START RUN 1iB
STOP ABNORMAL END: S/R MDS_READ_FIELD
STOP ABNORMAL END: S/R MDS_READ_FIELD
STOP ABNORMAL END: S/R MDS_READ_FIELD
mlx5: node669: got completion with error:
00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000
00000006 00000000 00000000 00000000
00000000 00008813 08003776 005016d2
[node669:mpi_rank_1][handle_cqe] Send desc error in msg to 5, wc_opcode=0
[node669:mpi_rank_1][handle_cqe] Msg from 5: wc.status=10, wc.wr_id=0x33b6dc0, wc.opcode=0, vbuf->phead->type=0 = MPIDI_CH3_PKT_EAGER_SEND
[node669:mpi_rank_1][handle_cqe] src/mpid/ch3/channels/mrail/src/gen2/ibv_channel_manager.c:548: [] Got completion with error 10, vendor code=0x88, dest rank=5
: No such file or directory (2)
==> RUN 1iB STOP without writing pickup => exit