/home/jm_c/test_engaging/MITgcm_o64Mpi/verification/exp4/tr_run.with_flt
../../../tools/tst_2+2 -mpi All -command "mpirun -np 4 -hostfile ../mpi_mfile.loc ./mitgcmuv"
 
cmdEXE='mpirun -np 4 -hostfile ../mpi_mfile.loc ./mitgcmuv'
from previous run STDOUT.0000, lastPick='       18 ckptA' ; iter='18' ; sufx='ckptA'
prepare parameter file 'data.tst' :
prepare file 'data.tst' : done
 diff data.tst data
36,39c36,37
<  nIter0=18,
<  nTimeSteps=4,
< # niter0=0,
< # nTimeSteps=18,
---
>  niter0=0,
>  nTimeSteps=18,
 
 link back: temp_tst/pickup*.ckptA*
 rename ckptA -> 0000000018 for all: pickup pickup_flt
  rnp_loc: pickup.ckptA pickup.0000000018
  rnp_loc: pickup_flt.ckptA pickup_flt.0000000018
 start-end iter: 18 , 20 , 22
 sufix: '0000000018' '0000000020' '0000000022'
 cmdEXE=mpirun -np 4 -hostfile ../mpi_mfile.loc ./mitgcmuv
==> START RUN 2 x 2 it
 STOP NORMAL END
 STOP
 STOP NORMAL END
 STOP
 STOP NORMAL END
 STOP
 STOP NORMAL END
 STOP
==>  END  RUN 2 x 2 it
 listP= pickup pickup_flt
  rnp_loc: pickup.ckptA pickup.0000000022
  rnp_loc: pickup_flt.ckptA pickup_flt.0000000022
 move_outp: res_2it
==> START RUN 1iA
mv: cannot stat ‘STDOUT.0000’: No such file or directory
srun: error: slurm_receive_msgs: [node337.inband:7818] failed: Socket timed out on send/recv operation
srun: error: Task launch for StepId=17112258.231 failed on node node337: Socket timed out on send/recv operation
srun: error: Application launch failed: Socket timed out on send/recv operation
srun: Job step aborted: Waiting up to 32 seconds for job step to finish.
[2026-07-04T02:30:55.035] error: *** STEP 17112258.231 ON node336 CANCELLED AT 2026-07-04T02:30:55 DUE to SIGNAL Killed ***
[mpiexec@node336] control_cb (pm/pmiserv/pmiserv_cb.c:200): assert (!closed) failed
[mpiexec@node336] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status
[mpiexec@node336] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:198): error waiting for event
[mpiexec@node336] main (ui/mpich/mpiexec.c:331): process manager error waiting for completion
empty or no output file: 'std_outp.1iA' => exit