/orcd/pool/004/jahn/centos7/MITgcm_gfortran-mpi/verification/global_ocean.90x40x15/run
../../../tools/tst_2+2 -mpi All -command "mpirun -n 5 -hostfile /tmp/machinefile.54697 ./mitgcmuv"
 
cmdEXE='mpirun -n 5 -hostfile /tmp/machinefile.54697 ./mitgcmuv'
from previous run STDOUT.0000, lastPick='    36010 ckptA' ; iter='36010' ; sufx='ckptA'
prepare parameter file 'data.tst' :
prepare file 'data.tst' : done
 diff data.tst data
50,53c50,51
<  nIter0=36010,
<  nTimeSteps=4,
< # nIter0=36000,
< # nTimeSteps=10,
---
>  nIter0=36000,
>  nTimeSteps=10,
63c61
< # pChkPtFreq= 311040000.,
---
>  pChkPtFreq= 311040000.,
 
 link back: temp_tst/pickup*.ckptA*
 rename ckptA -> 0000036010 for all: pickup_cd pickup
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036010
  rnp_loc: pickup.ckptA pickup.0000036010
 start-end iter: 36010 , 36012 , 36014
 sufix: '0000036010' '0000036012' '0000036014'
 cmdEXE=mpirun -n 5 -hostfile /tmp/machinefile.54697 ./mitgcmuv
==> START RUN 2 x 2 it
At line 2699 of file mdsio_write_field.f (unit = 9)
Fortran runtime error: Cannot open file 'pickup_cd.ckptA.004.005.data': No such file or directory

Error termination. Backtrace:
#0  0x2aaaab59d0cd in already_open
	at ../.././libgfortran/io/open.c:672
#1  0x51eddd in ???
#2  0x58af5b in ???
#3  0x4050b4 in ???
#4  0x6638c0 in ???
#5  0x6158c5 in ???
#6  0x6230b2 in ???
#7  0x65f0cf in ???
#8  0x67f47a in ???
#9  0x67f641 in ???
#10  0x5c79b6 in ???
#11  0x5c7c20 in ???
#12  0x2aaaaca47504 in ???
#13  0x402488 in ???
#14  0xffffffffffffffff in ???
[node669:mpi_rank_0][handle_cqe] Send desc error in msg to 3, wc_opcode=0
[node669:mpi_rank_0][handle_cqe] Msg from 3: wc.status=12, wc.wr_id=0x3581480, wc.opcode=0, vbuf->phead->type=0 = MPIDI_CH3_PKT_EAGER_SEND
[node669:mpi_rank_0][handle_cqe] src/mpid/ch3/channels/mrail/src/gen2/ibv_channel_manager.c:548: [] Got completion with error 12, vendor code=0x81, dest rank=3
: No such file or directory (2)
==>  END  RUN 2 x 2 it
 listP= pickup_cd pickup
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036014
  rnp_loc: pickup.ckptA pickup.0000036014
 move_outp: res_2it
==> START RUN 1iA
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
==>  END  RUN 1iA
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036012
  rnp_loc: pickup.ckptA pickup.0000036012
 move_outp: res_1iA
==> START RUN 1iB
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
STOP NORMAL END
==>  END  RUN 1iB
  rnp_loc: pickup_cd.ckptA pickup_cd.0000036014
  rnp_loc: pickup.ckptA pickup.0000036014
 move_outp: res_1iB
 
== compare cg2d_init_res :
 run 1iA:
   1.65751591607441E-02
   1.63278676916268E-02
 run 1iB:
   1.63747283872107E-02
   1.65245789689478E-02
 run 2it:
   1.61863808715843E-02
   1.63153374999570E-02
   1.63645560244180E-02
   1.65072448250086E-02
 
== diff pickup files : end of 1rst run (2x2 it) & end of 3rd run (2nd 2 it)
--> file=pickup.0000036014, listY=.001.001. .001.002. .001.003. .001.004. .002.001. .002.002. .002.003. .002.004. .003.001. .003.002. .003.003. .003.004. .004.001. .004.002. .004.003. .004.004. .005.001. .005.002. .005.003. .005.004. .006.001. .006.002. .006.003. .006.004. .007.001. .007.002. .007.003. .007.004.
 diff res_2it/pickup.0000036014.001.001.data res_1iB
Binary files res_2it/pickup.0000036014.001.001.data and res_1iB/pickup.0000036014.001.001.data differ
Diff outp= 2 (nb-Diff= 45414 ) ==> stop