margo-regression.qsub 4.6 KB
Newer Older
1
2
3
4
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
5
#COBALT -A CSC250STDM12
6
7
8
#COBALT -q debug-cache-quad

# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
9
10
# module unload xalt

11
12
13
# necessary when using the udreg option in Mercury
export MPICH_GNI_NDREG_ENTRIES=1024

14
15
module swap PrgEnv-intel PrgEnv-gnu
module load cce
16
17
18

. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg 
19
spack load -r bake
20
21
22

module list

Philip Carns's avatar
Philip Carns committed
23
export LD_LIBRARY_PATH=$LIBRARY_PATH
Philip Carns's avatar
Philip Carns committed
24

25
26
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"

Philip Carns's avatar
Philip Carns committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#ldd ./margo-p2p-latency

# NOTE: needed as of January 2018 to avoid conflicts between MPI and 
#       libfabric GNI provider
# NOTE: doing this with -e option to aprun
# NOTE: update as as of September 2018, this is no longer required now that 
#       mercury has switched back to alternative registration method for GNI
# export MPICH_GNI_NDREG_ENTRIES=2000

# echo "## MPI (one way, double the latency for round trip):"
# aprun -n 2 -N 1 ./osu_latency

echo "## Margo OFI/GNI (round trip):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000
echo "## Margo OFI/GNI (bw, 1MiB):"
42
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20
Philip Carns's avatar
Philip Carns committed
43
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
44
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20
Philip Carns's avatar
Philip Carns committed
45
echo "## Margo OFI/GNI (bw, 8MiB):"
46
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20
Philip Carns's avatar
Philip Carns committed
47
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
48
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20
Philip Carns's avatar
Philip Carns committed
49
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
50
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20
Philip Carns's avatar
Philip Carns committed
51
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
52
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20
Philip Carns's avatar
Philip Carns committed
53
54
55
56
57


echo "## Margo OFI/GNI (round trip, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
58
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
Philip Carns's avatar
Philip Carns committed
59
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
60
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
Philip Carns's avatar
Philip Carns committed
61
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
62
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
Philip Carns's avatar
Philip Carns committed
63
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
64
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20-t 0,0
Philip Carns's avatar
Philip Carns committed
65
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
66
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20-t 0,0
Philip Carns's avatar
Philip Carns committed
67
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
68
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
Philip Carns's avatar
Philip Carns committed
69
70
71
72

# echo "## Margo BMI/TCP (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# echo "## Margo BMI/TCP (bw):"
73
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
Philip Carns's avatar
Philip Carns committed
74
# echo "## Margo BMI/TCP (bw, 8x concurrency):"
75
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
Philip Carns's avatar
Philip Carns committed
76
77
78
79
80
81
82
83

# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000


# figure out nodes and hostnames for use with fi_
# declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
# server_hostname=$(printf "nid%05d\n" ${nodes[0]})
84

Philip Carns's avatar
Philip Carns committed
85
86
87
88
# echo "## fi_pingpong OFI/GNI (one way, double the usec/xfer for round trip):"
# aprun -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log &
# sleep 5
# aprun -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log &
89

Philip Carns's avatar
Philip Carns committed
90
91
# wait
# cat $COBALT_JOBID.fi_pingpong_client.log
92

Philip Carns's avatar
Philip Carns committed
93
94
95
# one more margo run, this time with diagnostics enabled
# echo "## Margo OFI/GNI (round trip) with diagnostics:"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -d $COBALT_JOBID.diag
96

Philip Carns's avatar
Philip Carns committed
97
98
# echo "## Mercury-runner OFI/GNI (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=ofi+gni://ipogif0:5000 h1