margo-regression.qsub 3.26 KB
Newer Older
1
2
3
4
5
6
7
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
#COBALT -A radix-io
#COBALT -q ibleaf3-debug

8
export HOME=$SANDBOX
Philip Carns's avatar
Philip Carns committed
9
10
# note: disable registration cache for verbs provider for now; see
#       discussion in https://github.com/ofiwg/libfabric/issues/5244
11
export FI_MR_CACHE_MAX_COUNT=0
12
13

. $SANDBOX/spack/share/spack/setup-env.sh
Philip Carns's avatar
Philip Carns committed
14
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
15
spack load -r ssg 
Philip Carns's avatar
Philip Carns committed
16
17
spack load -r $BAKE_STABLE_VER
spack load -r mvapich2
18
19
20
21
22
23

module list

# echo "## MPI (one way, double the latency for round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency

24
25
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 1"

26
27
28
sleep 1

echo "## Margo OFI/VERBS (round trip):"
29
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-latency -i 100000 -n "verbs://"
30
echo "## Margo OFI/VERBS (bw, 1MiB):"
31
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "verbs://" -c 1 -D 20
32
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency):"
33
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "verbs://" -c 8 -D 20
34
echo "## Margo OFI/VERBS (bw, 8MiB):"
35
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 1 -D 20
36
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency):"
37
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 8 -D 20
38
echo "## Margo OFI/VERBS (bw, 1MB unaligned):"
39
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 1 -D 20
40
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency):"
41
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 8 -D 20
42

43
44
45
sleep 1 

echo "## Margo OFI/VERBS (round trip, Hg busy spin):"
46
47
48
49
# see https://xgitlab.cels.anl.gov/sds/margo/issues/54 : 100000 iterations
# takes longer than 20 minutes, so cut that down a bit until we figure out
# what's going on.
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-latency -i 10000 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -t 0,0
50
echo "## Margo OFI/VERBS (bw, 1MiB, Hg busy spin):"
51
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 1 -D 20 -t 0,0
52
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency, Hg busy spin):"
53
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 8 -D 20 -t 0,0
54
echo "## Margo OFI/VERBS (bw, 8MiB, Hg busy spin):"
55
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 1 -D 20 -t 0,0
56
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency, Hg busy spin):"
57
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 8 -D 20 -t 0,0
58
echo "## Margo OFI/VERBS (bw, 1MB unaligned, Hg busy spin):"
59
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 1 -D 20 -t 0,0
60
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
61
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 8 -D 20 -t 0,0
62

63

64
65

# echo "## Mercury-runner CCI/VERBS (round trip):"
66
# mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=verbs;ofi_rxm:// h1