margo-regression-static.qsub 2.05 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
#COBALT -A ecp-testbed-01
#COBALT -q debug-cache-quad
#COBALT -M carns@mcs.anl.gov

# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
module unload xalt

# NOTE: needed as of January 2018 to avoid conflicts between MPI and 
#       libfabric GNI provider
# NOTE: doing this with -e option to aprun
# export MPICH_GNI_NDREG_ENTRIES=2000

echo "## MPI (one way, double the latency for round trip):"
aprun -n 2 -N 1 ./osu_latency

echo "## Margo OFI/GNI (round trip, busy spin):"
aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -t 0,0
echo "## Margo OFI/GNI (bw, busy spin):"
aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8x concurrency, busy spin):"
aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0

# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000

# figure out nodes and hostnames for use with fi_
declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
server_hostname=$(printf "nid%05d\n" ${nodes[0]})

echo "## fi_pingpong OFI/GNI (one way, double the usec/xfer for round trip):"
aprun -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log &
sleep 5
aprun -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log &

wait
cat $COBALT_JOBID.fi_pingpong_client.log

# one more margo run, this time with diagnostics enabled
# echo "## Margo OFI/GNI (round trip) with diagnostics:"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -d $COBALT_JOBID.diag

#echo "## Mercury-runner OFI/GNI (round trip):"
#aprun -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=ofi+gni://ipogif0:5000 h1