run_bgq.sh 1.84 KB
Newer Older
1 2 3 4
#!/bin/bash

NODES=8
PPN=8
5
NPROCS=$((NODES*PPN))
6
TARGET="/projects/visualization/ftessier/S3D"
7

8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=16777216
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=false

function updateSettings()
{
    printenv | egrep "TAPIOCA_"
    SETTINGS="TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
    SETTINGS="$SETTINGS TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
    SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
    SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
    SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
    SETTINGS="$SETTINGS TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
    SETTINGS="$SETTINGS TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"

    SETTINGS="$SETTINGS PAMID_VERBOSE=1"
    SETTINGS="$SETTINGS PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
    SETTINGS="$SETTINGS PAMID_COLLECTIVES=1"
    SETTINGS="$SETTINGS BG_SHAREDMEMSIZE=64"
    SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653"
    SETTINGS="$SETTINGS GPFSMPIO_BALANCECONTIG=1"
    SETTINGS="$SETTINGS GPFSMPIO_P2PCONTIG=1"
    #SETTINGS="$SETTINGS MPICH_MPIIO_HINTS=*:bg_nodes_pset=$TAPIOCA_NBAGGR"                                                                                   
    echo $SETTINGS
}


39 40
cd $HOME/TAPIOCA/examples/S3D-IO/run/
rm $TARGET/*
41
updateSettings
42 43

# Run and write out files with 8000pts/node
44 45 46
# cp -f ../input/s3d.in.110592_$NPROCS ../input/s3d.in
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./s3d_f90.x 
#> $NPROCS.s3d.out
47

48 49 50 51 52
# # Run and re-read ouput files written previously; check for errors
# cd  $NEW_WORKDIR/run
# cp -f ../input/s3d.in.8000_$PBS_NNODES.post ../input/s3d.in
# cp -f ../input/intmorph.in.8000_$PBS_NNODES ../input/intmorph.in
# aprun -n $PBS_NNODES ./s3d.x  >> $PBS_NNODES.s3d.out.$PBS_JOBID