margo-regression.qsub 3.02 KB
Newer Older
1 2 3 4 5 6 7
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
#COBALT -A radix-io
#COBALT -q ibleaf3-debug

8
get_pmdk_lib_path() {
9
        module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
10 11
}

12 13 14 15
export FI_FORK_UNSAFE=1

. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg 
Philip Carns's avatar
Philip Carns committed
16
spack load -r bake
17 18 19 20 21 22

module list

# TODO: why is this necessary?  We should be able to find gcc libraries...
export LD_LIBRARY_PATH="/soft/compilers/gcc/7.1.0/lib64:$LD_LIBRARY_PATH"

23 24 25 26
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"

27 28 29
# echo "## MPI (one way, double the latency for round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency

30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
sleep 1

echo "## Margo OFI/VERBS (round trip):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n "ofi+verbs;ofi_rxm://"
echo "## Margo OFI/VERBS (bw, 1MiB):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
echo "## Margo OFI/VERBS (bw, 8MiB):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
echo "## Margo OFI/VERBS (bw, 1MB unaligned):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20

47 48 49 50
sleep 1 

echo "## Margo OFI/VERBS (round trip, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -t 0,0
51
echo "## Margo OFI/VERBS (bw, 1MiB, Hg busy spin):"
52
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 1 -D 20 -t 0,0
53
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency, Hg busy spin):"
54
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 8 -D 20 -t 0,0
55 56 57 58 59 60 61 62
echo "## Margo OFI/VERBS (bw, 8MiB, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MB unaligned, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20 -t 0,0
63

64

65 66 67

# echo "## Mercury-runner CCI/VERBS (round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=verbs;ofi_rxm:// h1