bake-regression.sbatch 3.29 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
#!/bin/bash
#SBATCH -N 2
#SBATCH -A startup-carns
#SBATCH --ntasks-per-node=1
#SBATCH --time=15:00
#SBATCH -p bdwall

# example of running benchmarks with spack-compiled software stack

export LMOD_PAGER=none

module load numactl boost cmake

Philip Carns's avatar
Philip Carns committed
14
. $SANDBOX/spack/share/spack/setup-env.sh
15 16 17 18 19 20 21 22 23 24 25 26

spack load -r ssg
spack load -r bake

module list

# force Intel MPI to use Infiniband instead of Omnipath/PSM2.  If we let 
# Intel MPI use it's preferred PSM2 path then Mercury can't open interface
# because of apparent MR_SCALABLE/MR_BASIC conflict
# we should also try MR_SCALABLE in Mercury NA as a possible workaround
export I_MPI_FABRICS=shm:ofa

27 28
echo ### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on bebop nodes: cores 18-35

29 30 31
echo "## Bake OFI/PSM2 (bdw):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
32
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 
33 34 35 36

echo "## Bake OFI/PSM2 (8x concurrency, bdw):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
37
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8
38 39 40 41

echo "## Bake OFI/PSM2 (bdw, Hg busy spin):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
42
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -t 0,0
43 44 45 46

echo "## Bake OFI/PSM2 (8x concurrency, bdw, Hg busy spin):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
47
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -t 0,0
48

49 50 51 52

echo "## Bake OFI/PSM2 (bdw, 8 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
53
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 8
54 55 56 57

echo "## Bake OFI/PSM2 (8x concurrency, bdw, 8 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
58
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 8
59 60 61 62

echo "## Bake OFI/PSM2 (bdw, Hg busy spin, 8 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
63
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -t 0,0 -r 8
64 65 66 67

echo "## Bake OFI/PSM2 (8x concurrency, bdw, Hg busy spin, 8 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
68
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -t 0,0 -r 8
69

70 71 72
echo "## Bake OFI/PSM2 (8x concurrency, bdw, 8 rpc es, pipeline 4M-16-32):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
73
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 8 -i 4194304,16,32
74 75 76 77

echo "## Bake OFI/PSM2 (8x concurrency, bdw, 8 rpc es, pipeline 4M-16-2):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
78
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 8 -i 4194304,16,2
79 80 81 82

echo "## Bake OFI/PSM2 (8x concurrency, bdw, 8 rpc es, pipeline 16M-8-1):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
83
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 8 -i 16777216,8,1