/orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_oce_latlon/tr_run.ebm ../../../tools/tst_2+2 -mpi All -command "mpirun -env I_MPI_DEBUG 2 -n 6 ./mitgcmuv" cmdEXE='mpirun -env I_MPI_DEBUG 2 -n 6 ./mitgcmuv' from previous run STDOUT.0000, lastPick=' 20 ckptA' ; iter='20' ; sufx='ckptA' prepare parameter file 'data.tst' : prepare file 'data.tst' : done diff data.tst data 39,42c39,40 < nIter0=20, < nTimeSteps=4, < # nIter0 = 0, < # nTimeSteps = 20, --- > nIter0 = 0, > nTimeSteps = 20, 52c50 < # pChkptFreq= 311040000., --- > pChkptFreq= 311040000., link back: temp_tst/pickup*.ckptA* rename ckptA -> 0000000020 for all: pickup_cd pickup rnp_loc: pickup_cd.ckptA pickup_cd.0000000020 rnp_loc: pickup.ckptA pickup.0000000020 start-end iter: 20 , 22 , 24 sufix: '0000000020' '0000000022' '0000000024' cmdEXE=mpirun -env I_MPI_DEBUG 2 -n 6 ./mitgcmuv ==> START RUN 2 x 2 it [0] MPI startup(): Single-threaded optimized library [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [5] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [5] MPI startup(): shm and dapl data transfer modes SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 NH 10.3215387668278 25.5650261190031 SH 5.28727219584948 23.3261502700456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 NH 10.3110701256353 25.5641420356456 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 NH 10.3000310343672 25.5631333463182 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 NH 10.2884779126362 25.5620269288389 NORMAL END NORMAL END NORMAL END NORMAL END NORMAL END NORMAL END ==> END RUN 2 x 2 it listP= pickup_cd pickup rnp_loc: pickup_cd.ckptA pickup_cd.0000000024 rnp_loc: pickup.ckptA pickup.0000000024 move_outp: res_2it ==> START RUN 1iA [0] MPI startup(): Single-threaded optimized library [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [5] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): shm and dapl data transfer modes [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): shm and dapl data transfer modes SH 5.28883233517024 23.3242480895835 NH 10.3224176280368 25.5594397838430 SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 NH 10.3224176280368 25.5594397838430 SH 5.28883233517024 23.3242480895835 NH 10.3224176280368 25.5594397838430 SH 5.28883233517024 23.3242480895835 NH 10.3224176280368 25.5594397838430 NH 10.3224176280368 25.5594397838430 NH 10.3224176280368 25.5594397838430 SH 5.28727219584948 23.3261210947743 NH 10.3105682700896 25.5586199150837 SH 5.28727219584948 23.3261210947743 NH 10.3105682700896 25.5586199150837 SH 5.28727219584948 23.3261210947743 NH 10.3105682700896 25.5586199150837 SH 5.28727219584948 23.3261210947743 NH 10.3105682700896 25.5586199150837 SH 5.28727219584948 23.3261210947743 NH 10.3105682700896 25.5586199150837 SH 5.28727219584948 23.3261210947743 NH 10.3105682700896 25.5586199150837 NORMAL END NORMAL END NORMAL END NORMAL END NORMAL END NORMAL END ==> END RUN 1iA rnp_loc: pickup_cd.ckptA pickup_cd.0000000022 rnp_loc: pickup.ckptA pickup.0000000022 move_outp: res_1iA ==> START RUN 1iB [0] MPI startup(): Single-threaded optimized library [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [5] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [5] MPI startup(): shm and dapl data transfer modes ABNORMAL END: S/R MDS_READ_FIELD ==> RUN 1iB STOP without writing pickup => exit