/orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_ocean.90x40x15/run ../../../tools/tst_2+2 -mpi All -command "mpirun -env I_MPI_DEBUG 2 -n 5 ./mitgcmuv" cmdEXE='mpirun -env I_MPI_DEBUG 2 -n 5 ./mitgcmuv' from previous run STDOUT.0000, lastPick=' 36010 ckptA' ; iter='36010' ; sufx='ckptA' prepare parameter file 'data.tst' : prepare file 'data.tst' : done diff data.tst data 50,53c50,51 < nIter0=36010, < nTimeSteps=4, < # nIter0=36000, < # nTimeSteps=10, --- > nIter0=36000, > nTimeSteps=10, 63c61 < # pChkPtFreq= 311040000., --- > pChkPtFreq= 311040000., link back: temp_tst/pickup*.ckptA* rename ckptA -> 0000036010 for all: pickup_cd pickup rnp_loc: pickup_cd.ckptA pickup_cd.0000036010 rnp_loc: pickup.ckptA pickup.0000036010 start-end iter: 36010 , 36012 , 36014 sufix: '0000036010' '0000036012' '0000036014' cmdEXE=mpirun -env I_MPI_DEBUG 2 -n 5 ./mitgcmuv ==> START RUN 2 x 2 it [0] MPI startup(): Single-threaded optimized library [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): shm and dapl data transfer modes NORMAL END NORMAL END NORMAL END NORMAL END NORMAL END ==> END RUN 2 x 2 it listP= pickup_cd pickup rnp_loc: pickup_cd.ckptA pickup_cd.0000036014 rnp_loc: pickup.ckptA pickup.0000036014 move_outp: res_2it ==> START RUN 1iA [0] MPI startup(): Single-threaded optimized library [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): shm and dapl data transfer modes NORMAL END NORMAL END NORMAL END NORMAL END NORMAL END ==> END RUN 1iA rnp_loc: pickup_cd.ckptA pickup_cd.0000036012 rnp_loc: pickup.ckptA pickup.0000036012 move_outp: res_1iA ==> START RUN 1iB [0] MPI startup(): Single-threaded optimized library [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): shm and dapl data transfer modes [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): shm and dapl data transfer modes ABNORMAL END: S/R MDS_READ_FIELD ==> RUN 1iB STOP without writing pickup => exit