#PSUB -s /bin/bash  # Sets your shell in batch
#PSUB -c alc       # Where to run the job

#PSUB -eo          # Send std error & std out to the same file

#PSUB -ln  $NUM       # Number of nodes to use
#PSUB -g   $NUM       # Total Number of tasks to use
#PSUB -cpn 1       # cpus per node

####PSUB -c 1024Mb     # memory limit
#PSUB -lc 1500        # Core file size per process
#PSUB -nr          # Do not automatically resubmit job
#PSUB -tM 20m      # Select time limit. The default time limit
                   # is only 30 minutes! Time can be HH:MM:SS or HH:MM

#PSUB -o $CWD/$OUT   # filename for output

# Put your commands here. Remember to 'cd' to the appropriate
# directory, because the job will initially be in your home directory.
# To run a parallel job, you need to use the srun.



echo job $PSUB_JOBID nodes $NUM name $NAME

# environment
cd $CWD
export LD_LIBRARY_PATH=/usr/lib/mpi/mpi_gnu/lib

# create fakestore dirs
srun -l -N $NUM -ppbatch bash -c "test -d tmp/osddata || mkdir tmp/osddata || echo cant make osddata  ; uptime"

# go
srun -l -N $NUM -ppbatch $CMD && touch $DONE

# clean up fakestore
srun -l -N $NUM -ppbatch bash -c 'uptime ; rm -r tmp/osddata/*'