/orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_oce_latlon/tr_run.ebm ../../../tools/tst_2+2 -mpi All -command "mpirun -env I_MPI_DEBUG 2 -n 6 ./mitgcmuv" cmdEXE='mpirun -env I_MPI_DEBUG 2 -n 6 ./mitgcmuv' from previous run STDOUT.0000, lastPick=' 20 ckptA' ; iter='20' ; sufx='ckptA' prepare parameter file 'data.tst' : prepare file 'data.tst' : done diff data.tst data 39,42c39,40 < nIter0=20, < nTimeSteps=4, < # nIter0 = 0, < # nTimeSteps = 20, --- > nIter0 = 0, > nTimeSteps = 20, 52c50 < # pChkptFreq= 311040000., --- > pChkptFreq= 311040000., link back: temp_tst/pickup*.ckptA* rename ckptA -> 0000000020 for all: pickup_cd pickup rnp_loc: pickup_cd.ckptA pickup_cd.0000000020 rnp_loc: pickup.ckptA pickup.0000000020 start-end iter: 20 , 22 , 24 sufix: '0000000020' '0000000022' '0000000024' cmdEXE=mpirun -env I_MPI_DEBUG 2 -n 6 ./mitgcmuv ==> START RUN 2 x 2 it [0] MPI startup(): Single-threaded optimized library [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [5] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [5] MPI startup(): shm and dapl data transfer modes SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 SH 5.28883233517024 23.3242480895835 NH 10.3215387668278 25.5650261190031 NH 10.3215387668278 25.5650261190031 NH 10.3215387668278 25.5650261190031 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 SH 5.28727219584948 23.3261502700456 NH 10.3110701256353 25.5641420356456 NH 10.3110701256353 25.5641420356456 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 SH 5.28470884825873 23.3280517619924 SH 5.28470884825873 23.3280517619924 NH 10.3000310343672 25.5631333463182 NH 10.3000310343672 25.5631333463182 NH 10.3000310343672 25.5631333463182 SH 5.28115412988211 23.3299307278742 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 NH 10.2884779126362 25.5620269288389 SH 5.28115412988211 23.3299307278742 NH 10.2884779126362 25.5620269288389 NH 10.2884779126362 25.5620269288389 forrtl: No such file or directory forrtl: severe (29): file not found, unit 9, file /orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_oce_latlon/tr_run.ebm/pickup_cd.ckptA.002.002.data Image PC Routine Line Source libifcoremt.so.5 00002AAAACEC91C2 for__io_return Unknown Unknown libifcoremt.so.5 00002AAAACEED2DF for_open Unknown Unknown mitgcmuv 00000000004EA3D9 Unknown Unknown Unknown mitgcmuv 000000000055A795 Unknown Unknown Unknown mitgcmuv 0000000000409E62 Unknown Unknown Unknown mitgcmuv 0000000000667A96 Unknown Unknown Unknown mitgcmuv 0000000000617A74 Unknown Unknown Unknown mitgcmuv 000000000062CA73 Unknown Unknown Unknown mitgcmuv 00000000006635C0 Unknown Unknown Unknown mitgcmuv 0000000000687783 Unknown Unknown Unknown mitgcmuv 0000000000687A29 Unknown Unknown Unknown mitgcmuv 00000000005C7EE4 Unknown Unknown Unknown mitgcmuv 0000000000402D2E Unknown Unknown Unknown libc-2.17.so 00002AAAAF3EE505 __libc_start_main Unknown Unknown mitgcmuv 0000000000402C39 Unknown Unknown Unknown ==> END RUN 2 x 2 it listP= pickup_cd pickup rnp_loc: pickup_cd.ckptA pickup_cd.0000000024 rnp_loc: pickup.ckptA pickup.0000000024 move_outp: res_2it ==> START RUN 1iA [0] MPI startup(): Single-threaded optimized library [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [5] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [5] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): shm and dapl data transfer modes SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 NH 10.2827839603725 25.5727363898283 SH 5.28883233517024 23.3242480895835 SH 5.28883233517024 23.3242480895835 NH 10.2827839603725 25.5727363898283 SH 5.28883233517024 23.3242480895835 NH 10.2827839603725 25.5727363898283 SH 5.28883233517024 23.3242480895835 NH 10.2827839603725 25.5727363898283 NH 10.2827839603725 25.5727363898283 NH 10.2827839603725 25.5727363898283 SH 5.28727219584948 23.3269669040565 NH 10.2712785268694 25.5717967645230 SH 5.28727219584948 23.3269669040565 NH 10.2712785268694 25.5717967645230 SH 5.28727219584948 23.3269669040565 NH 10.2712785268694 25.5717967645230 SH 5.28727219584948 23.3269669040565 SH 5.28727219584948 23.3269669040565 SH 5.28727219584948 23.3269669040565 NH 10.2712785268694 25.5717967645230 NH 10.2712785268694 25.5717967645230 NH 10.2712785268694 25.5717967645230 forrtl: No such file or directory forrtl: severe (29): file not found, unit 9, file /orcd/pool/004/jahn/centos7/MITgcm_ifort-mpi-fast_san/verification/global_oce_latlon/tr_run.ebm/pickup.ckptA.003.002.data Image PC Routine Line Source libifcoremt.so.5 00002AAAACEC91C2 for__io_return Unknown Unknown libifcoremt.so.5 00002AAAACEED2DF for_open Unknown Unknown mitgcmuv 00000000004EA3D9 Unknown Unknown Unknown mitgcmuv 000000000055A795 Unknown Unknown Unknown mitgcmuv 000000000068FA4C Unknown Unknown Unknown mitgcmuv 0000000000617AB5 Unknown Unknown Unknown mitgcmuv 000000000062CA73 Unknown Unknown Unknown mitgcmuv 00000000006635C0 Unknown Unknown Unknown mitgcmuv 0000000000687783 Unknown Unknown Unknown mitgcmuv 0000000000687A29 Unknown Unknown Unknown mitgcmuv 00000000005C7EE4 Unknown Unknown Unknown mitgcmuv 0000000000402D2E Unknown Unknown Unknown libc-2.17.so 00002AAAAF3EE505 __libc_start_main Unknown Unknown mitgcmuv 0000000000402C39 Unknown Unknown Unknown ==> END RUN 1iA rnp_loc: pickup_cd.ckptA pickup_cd.0000000022 rnp_loc: pickup.ckptA pickup.0000000022 move_outp: res_1iA ==> START RUN 1iB [0] MPI startup(): Single-threaded optimized library [0] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [2] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [1] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [0] MPI startup(): shm and dapl data transfer modes [2] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [2] MPI startup(): shm and dapl data transfer modes [1] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [1] MPI startup(): shm and dapl data transfer modes [5] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [3] DAPL startup(): trying to open DAPL provider from I_MPI_DAPL_PROVIDER: ofa-v2-mlx5_0-1u [4] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [5] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [4] MPI startup(): shm and dapl data transfer modes [5] MPI startup(): shm and dapl data transfer modes [3] MPI startup(): DAPL provider ofa-v2-mlx5_0-1u [3] MPI startup(): shm and dapl data transfer modes ABNORMAL END: S/R MDS_READ_FIELD ABNORMAL END: S/R MDS_READ_FIELD ==> RUN 1iB STOP without writing pickup => exit