/orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_ocean.90x40x15/tr_run.idemix ../../../tools/tst_2+2 -mpi All -command "mpirun -env I_MPI_DEBUG 2 -n 5 ./mitgcmuv" cmdEXE='mpirun -env I_MPI_DEBUG 2 -n 5 ./mitgcmuv' from previous run STDOUT.0000, lastPick=' 10 ckptA' ; iter='10' ; sufx='ckptA' prepare parameter file 'data.tst' : prepare file 'data.tst' : done diff data.tst data 46,49c46,47 < nIter0=10, < nTimeSteps=4, < # nIter0=0, < # nTimeSteps=10, --- > nIter0=0, > nTimeSteps=10, 59c57 < # pChkPtFreq= 311040000., --- > pChkPtFreq= 311040000., link back: temp_tst/pickup*.ckptA* rename ckptA -> 0000000010 for all: pickup_cd pickup pickup_ggl90 rnp_loc: pickup_cd.ckptA pickup_cd.0000000010 rnp_loc: pickup.ckptA pickup.0000000010 rnp_loc: pickup_ggl90.ckptA pickup_ggl90.0000000010 start-end iter: 10 , 12 , 14 sufix: '0000000010' '0000000012' '0000000014' cmdEXE=mpirun -env I_MPI_DEBUG 2 -n 5 ./mitgcmuv ==> START RUN 2 x 2 it [0] MPI startup(): Single-threaded optimized library [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): shm and dapl data transfer modes [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes forrtl: No such file or directory forrtl: severe (29): file not found, unit 9, file /orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_ocean.90x40x15/tr_run.idemix/pickup_cd.ckptA.007.005.data Image PC Routine Line Source libifcoremt.so.5 00002AAAACEC91C2 for__io_return Unknown Unknown libifcoremt.so.5 00002AAAACEED2DF for_open Unknown Unknown mitgcmuv 00000000005A3772 Unknown Unknown Unknown mitgcmuv 000000000063FEE5 Unknown Unknown Unknown mitgcmuv 0000000000405A56 Unknown Unknown Unknown mitgcmuv 00000000007D1DE6 Unknown Unknown Unknown mitgcmuv 0000000000753F84 Unknown Unknown Unknown mitgcmuv 000000000076F8AB Unknown Unknown Unknown mitgcmuv 00000000007CD940 Unknown Unknown Unknown mitgcmuv 00000000007FFB4B Unknown Unknown Unknown mitgcmuv 00000000007FFD09 Unknown Unknown Unknown mitgcmuv 00000000006B8D84 Unknown Unknown Unknown mitgcmuv 00000000004028BE Unknown Unknown Unknown libc-2.17.so 00002AAAAF3EE505 __libc_start_main Unknown Unknown mitgcmuv 00000000004027C9 Unknown Unknown Unknown forrtl: No such file or directory ==> END RUN 2 x 2 it listP= pickup_cd pickup pickup_ggl90 rnp_loc: pickup_cd.ckptA pickup_cd.0000000014 rnp_loc: pickup.ckptA pickup.0000000014 rnp_loc: pickup_ggl90.ckptA pickup_ggl90.0000000014 move_outp: res_2it ==> START RUN 1iA [0] MPI startup(): Single-threaded optimized library [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes forrtl: No such file or directory forrtl: severe (29): file not found, unit 9, file /orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_ocean.90x40x15/tr_run.idemix/pickup_cd.ckptA.001.005.data Image PC Routine Line Source libifcoremt.so.5 00002AAAACEC91C2 for__io_return Unknown Unknown libifcoremt.so.5 00002AAAACEED2DF for_open Unknown Unknown mitgcmuv 00000000005A3772 Unknown Unknown Unknown mitgcmuv 000000000063FEE5 Unknown Unknown Unknown mitgcmuv 0000000000405A18 Unknown Unknown Unknown mitgcmuv 00000000007D1DE6 Unknown Unknown Unknown mitgcmuv 0000000000753F84 Unknown Unknown Unknown mitgcmuv 000000000076F8AB Unknown Unknown Unknown mitgcmuv 00000000007CD940 Unknown Unknown Unknown mitgcmuv 00000000007FFB4B Unknown Unknown Unknown mitgcmuv 00000000007FFD09 Unknown Unknown Unknown mitgcmuv 00000000006B8D84 Unknown Unknown Unknown mitgcmuv 00000000004028BE Unknown Unknown Unknown libc-2.17.so 00002AAAAF3EE505 __libc_start_main Unknown Unknown mitgcmuv 00000000004027C9 Unknown Unknown Unknown ==> END RUN 1iA rnp_loc: pickup_cd.ckptA pickup_cd.0000000012 rnp_loc: pickup.ckptA pickup.0000000012 rnp_loc: pickup_ggl90.ckptA pickup_ggl90.0000000012 move_outp: res_1iA ==> START RUN 1iB [0] MPI startup(): Single-threaded optimized library [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes ABNORMAL END: S/R READ_PICKUP ABNORMAL END: S/R READ_PICKUP ABNORMAL END: S/R READ_PICKUP ABNORMAL END: S/R READ_PICKUP ABNORMAL END: S/R READ_PICKUP ==> RUN 1iB STOP without writing pickup => exit