bake-regression.sbatch 2.05 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
#!/bin/bash
#SBATCH -N 2
#SBATCH -A startup-carns
#SBATCH --ntasks-per-node=1
#SBATCH --time=15:00
#SBATCH -p bdwall

# example of running benchmarks with spack-compiled software stack

export LMOD_PAGER=none

module load numactl boost cmake

Philip Carns's avatar
Philip Carns committed
14
. $SANDBOX/spack/share/spack/setup-env.sh
15 16 17 18 19 20 21 22 23 24 25 26

spack load -r ssg
spack load -r bake

module list

# force Intel MPI to use Infiniband instead of Omnipath/PSM2.  If we let 
# Intel MPI use it's preferred PSM2 path then Mercury can't open interface
# because of apparent MR_SCALABLE/MR_BASIC conflict
# we should also try MR_SCALABLE in Mercury NA as a possible workaround
export I_MPI_FABRICS=shm:ofa

Philip Carns's avatar
Philip Carns committed
27
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on bebop nodes: cores 18-35"
28

29 30 31
echo "## Bake OFI/PSM2 (bdw):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
32
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 
33 34 35 36

echo "## Bake OFI/PSM2 (8x concurrency, bdw):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
37
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8
38

Philip Carns's avatar
Philip Carns committed
39
echo "## Bake OFI/PSM2 (bdw, 12 rpc es):"
40 41
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
Philip Carns's avatar
Philip Carns committed
42
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 12
43

Philip Carns's avatar
Philip Carns committed
44
echo "## Bake OFI/PSM2 (8x concurrency, bdw, 12 rpc es):"
45 46
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
Philip Carns's avatar
Philip Carns committed
47
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 12
48

Philip Carns's avatar
Philip Carns committed
49
echo "## Bake OFI/PSM2 (bdw, 12 rpc es, pipelining):"
50 51
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
Philip Carns's avatar
Philip Carns committed
52
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 12 -i
53

Philip Carns's avatar
Philip Carns committed
54
echo "## Bake OFI/PSM2 (8x concurrency, bdw, 12 rpc es, pipelining):"
55 56
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
Philip Carns's avatar
Philip Carns committed
57
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 12 -i
58