#!/bin/csh
#
# Example PBS script to run a job on the myrinet-3 cluster.
# The lines beginning #PBS set various queuing parameters.
#
# o -N Job Name
#PBS -N fizhi
#
#
# o -l resource lists that control where job goes
#      here we ask for 3 nodes, each with the attribute "p4".
#PBS -l nodes=6:p4
#
#PBS -q four-twelve
#
# o Where to write output
# #PBS -e TTT.err.$$ 
#
# #PBS -o TTT.out.$$
#
#
# o Export all my environment variables to the job
#PBS -V
#

echo 'start'
#----------------------------------------------
set namPBS=fizhi
set runDir=/SCRATCH/s04/molod/fizhi-cs-32x32x40/input
set executable=/SCRATCH/s04/molod/fizhi-cs-32x32x40/build/mitgcmuv
#-------------------
# echo 'run in dir :' $runDir
cd $runDir

set its=`egrep 'nIter0' data | sed 's/ nIter0=//'| sed 's/,$//'`

set numPBS=`echo $PBS_NODEFILE | sed 's/\/usr\/spool\/PBS\/aux\///g'| sed 's/\.cg01//g'`

#-- avoid any output before this point !
#----------------------------------------------
#echo $PBS_NODEFILE
#cat  $PBS_NODEFILE
#echo 'The list above shows the nodes this job has exclusive access to.'
#echo 'The list can be found in the file named in the variable $PBS_NODEFILE'

rm -f run_here
date > run_here
cat $PBS_NODEFILE >> run_here
echo 'PBS job Number:' $numPBS >> run_here
echo on `hostname` run $executable 'in dir:' $runDir >> run_here

#-------------------
rm -f IamReady
echo on `hostname` run $executable 'in dir:' `pwd`

#-------------------
cp -p run_here std_outp
echo ' ' >> std_outp

cat $PBS_NODEFILE > mf
set ncpus = ( `wc -l mf | awk '{print $1}'` )
if ( $ncpus == 1 ) then
 $executable >>& std_outp
 set out=$?
 echo 'end with status' $out
else
 /usr/local/pkg/mpi/mpi-1.2.4..8a-gm-1.5/pgi/bin/mpirun.ch_gm -machinefile mf --gm-kill 5 -v -np $ncpus $executable
 set out=$?
 echo 'end with status' $out
 set out=0
endif

echo 'Copying datetime file to datetime0 - time is: ' `cat datetime`
cp datetime datetime0

set ite=`grep CHECKPOINT STDOUT.0000 | grep ckptA | sed 's/(PID.TID 0000.0001) %CHECKPOINT//' | sed 's/ //g' | sed 's/ckptA//'`
set nitPick = `/home/molod/bin/add0upto10c $ite`
if ( -f pickup.ckptA.001.001.data ) then
 echo 'file:' pickup.ckptA.001.001.data 'exist'
 if ( $out == 0 ) then
#- prepare new submission :
 echo 'Copy pickup ckptA files to pickup ' $nitPick 'files'
   mv pickup.ckptA.001.001.data pickup.$nitPick.001.001.data
   mv pickup.ckptA.002.001.data pickup.$nitPick.002.001.data
   mv pickup.ckptA.003.001.data pickup.$nitPick.003.001.data
   mv pickup.ckptA.004.001.data pickup.$nitPick.004.001.data
   mv pickup.ckptA.005.001.data pickup.$nitPick.005.001.data
   mv pickup.ckptA.006.001.data pickup.$nitPick.006.001.data
   mv pickup.ckptA.001.001.meta pickup.$nitPick.001.001.meta
   mv pickup.ckptA.002.001.meta pickup.$nitPick.002.001.meta
   mv pickup.ckptA.003.001.meta pickup.$nitPick.003.001.meta
   mv pickup.ckptA.004.001.meta pickup.$nitPick.004.001.meta
   mv pickup.ckptA.005.001.meta pickup.$nitPick.005.001.meta
   mv pickup.ckptA.006.001.meta pickup.$nitPick.006.001.meta
   mv pickup_fizhi.ckptA.001.001.data pickup_fizhi.$nitPick.001.001.data
   mv pickup_fizhi.ckptA.002.001.data pickup_fizhi.$nitPick.002.001.data
   mv pickup_fizhi.ckptA.003.001.data pickup_fizhi.$nitPick.003.001.data
   mv pickup_fizhi.ckptA.004.001.data pickup_fizhi.$nitPick.004.001.data
   mv pickup_fizhi.ckptA.005.001.data pickup_fizhi.$nitPick.005.001.data
   mv pickup_fizhi.ckptA.006.001.data pickup_fizhi.$nitPick.006.001.data
   mv pickup_fizhi.ckptA.001.001.meta pickup_fizhi.$nitPick.001.001.meta
   mv pickup_fizhi.ckptA.002.001.meta pickup_fizhi.$nitPick.002.001.meta
   mv pickup_fizhi.ckptA.003.001.meta pickup_fizhi.$nitPick.003.001.meta
   mv pickup_fizhi.ckptA.004.001.meta pickup_fizhi.$nitPick.004.001.meta
   mv pickup_fizhi.ckptA.005.001.meta pickup_fizhi.$nitPick.005.001.meta
   mv pickup_fizhi.ckptA.006.001.meta pickup_fizhi.$nitPick.006.001.meta
   mv pickup_vegtiles.ckptA.001.001.data pickup_vegtiles.$nitPick.001.001.data
   mv pickup_vegtiles.ckptA.002.001.data pickup_vegtiles.$nitPick.002.001.data
   mv pickup_vegtiles.ckptA.003.001.data pickup_vegtiles.$nitPick.003.001.data
   mv pickup_vegtiles.ckptA.004.001.data pickup_vegtiles.$nitPick.004.001.data
   mv pickup_vegtiles.ckptA.005.001.data pickup_vegtiles.$nitPick.005.001.data
   mv pickup_vegtiles.ckptA.006.001.data pickup_vegtiles.$nitPick.006.001.data
 echo 'Change start iteration number for next run '
   sed "s/ nIter0=$its/ nIter0=$ite/" data > data.tmp.$numPBS
   mv data.tmp.$numPBS data
   sleep 5
 endif
else
 echo 'file:' pickup.ckptA.001.001.data 'do not exist !'
 set out=9
endif

if ( $out != 0 ) then
 echo 'bad return status => STOP here'
 exit $out
endif 

 touch IamReady
  qsub runScript

exit
