bake-regression.qsub 2.49 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
#COBALT -A radix-io
#COBALT -q debug-cache-quad

# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt

module swap PrgEnv-intel PrgEnv-gnu
module load cce

. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg 
Philip Carns's avatar
bug fix  
Philip Carns committed
16
spack load -r bake
17 18 19

module list

Philip Carns's avatar
Philip Carns committed
20 21 22
# NOTE: rpath doesn't seem to be set correctly, and the paths we need are 
#  in LIBRARY_PATH instead of LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$LIBRARY_PATH
23

24 25 26 27 28 29 30 31 32 33 34
# find nodes in job.  We have to do this so that we can manually specify 
# in each aprun so that server ranks consitently run on node where we
# set up storage space
declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));

echo "## testing launcher placement:"
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} hostname

Philip Carns's avatar
Philip Carns committed
35
echo "## Bake OFI/GNI:"
36 37 38 39 40
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 
41

Philip Carns's avatar
Philip Carns committed
42
echo "## Bake OFI/GNI (8x concurrency):"
43 44 45 46 47
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 
48

Philip Carns's avatar
Philip Carns committed
49
echo "## Bake OFI/GNI (Hg busy spin):"
50 51 52 53 54
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 -t 0,0
55

Philip Carns's avatar
Philip Carns committed
56
echo "## Bake OFI/GNI (8x concurrency, Hg busy spin):"
57 58 59 60 61
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 -t 0,0
62