Commit 04bf9aed authored by Philip Carns's avatar Philip Carns

make regression tests consistent about numactl

parent ab61c33e
......@@ -21,35 +21,35 @@ module list
# make sure that MPI and libfabric can share PSM2
export PSM2_MULTI_EP=1
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on bebop nodes: cores 18-35"
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
echo "## Bake OFI/PSM2 (bdw):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1
srun numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1
echo "## Bake OFI/PSM2 (8x concurrency, bdw):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8
srun numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8
echo "## Bake OFI/PSM2 (bdw, 12 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 12
srun numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 12
echo "## Bake OFI/PSM2 (8x concurrency, bdw, 12 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 12
srun numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 12
echo "## Bake OFI/PSM2 (bdw, 12 rpc es, pipelining):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 12 -i
srun numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 1 -r 12 -i
echo "## Bake OFI/PSM2 (8x concurrency, bdw, 12 rpc es, pipelining):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
srun taskset -c 18-35 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 12 -i
srun numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n psm2://enp6s0f0:5000 -p /dev/shm/foo.dat -c 8 -r 12 -i
......@@ -21,41 +21,43 @@ module list
# make sure that MPI and libfabric can share PSM2
export PSM2_MULTI_EP=1
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
echo "## Margo OFI/PSM2 (round trip, bdw):"
mpirun ./margo-p2p-latency -i 100000 -n psm2://enp6s0f0:5000
mpirun numactl -N 0 -m 0 ./margo-p2p-latency -i 100000 -n psm2://enp6s0f0:5000
echo "## Margo OFI/PSM2 (bw, 1MiB, bdw):"
mpirun ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 1 -D 20
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 1 -D 20
echo "## Margo OFI/PSM2 (bw, 1MiB, 8x concurrency, bdw):"
mpirun ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 8 -D 20
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 8 -D 20
echo "## Margo OFI/PSM2 (bw, 8MiB, bdw):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20
echo "## Margo OFI/PSM2 (bw, 8MiB, 8x concurrency, bdw):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20
echo "## Margo OFI/PSM2 (bw, 1MB unaligned, bdw):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 1 -D 20
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 1 -D 20
echo "## Margo OFI/PSM2 (bw, 1MB unaligned, 8x concurrency, bdw):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 8 -D 20
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 8 -D 20
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, bdw):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -m /dev/shm/foo
# mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -m /dev/shm/foo
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, 8x concurrency, bdw):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -m /dev/shm/foo
# mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -m /dev/shm/foo
echo "## Margo OFI/PSM2 (round trip, bdw, Hg busy spin):"
mpirun ./margo-p2p-latency -i 100000 -n psm2://enp6s0f0:5000 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-latency -i 100000 -n psm2://enp6s0f0:5000 -t 0,0
echo "## Margo OFI/PSM2 (bw, 1MiB, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 1MiB, 8x concurrency, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 8MiB, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 8MiB, 8x concurrency, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 1MB unaligned, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 1MB unaligned, 8x concurrency, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, bdw, Hg busy spin):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0 -m /dev/shm/foo
# mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0 -m /dev/shm/foo
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, 8x concurrency, bdw, Hg busy spin):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0 -m /dev/shm/foo
# mpirun numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0 -m /dev/shm/foo
......@@ -15,35 +15,35 @@ spack load -r mvapich2
module list
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11"
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 1"
echo "## Bake OFI/VERBS:"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 1
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 1
echo "## Bake OFI/VERBS (8x concurrency):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 8
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 8
echo "## Bake OFI/VERBS (4 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 1 -r 4
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 1 -r 4
echo "## Bake OFI/VERBS (8x concurrency, 4 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 8 -r 4
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 8 -r 4
echo "## Bake OFI/VERBS (4 rpc es, pipelining):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 1 -r 4 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 1 -r 4 -i
echo "## Bake OFI/VERBS (8x concurrency, 4 rpc es, pipelining):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 8 -r 4 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+verbs;ofi_rxm://" -p /dev/shm/foo.dat -c 8 -r 4 -i
......@@ -17,41 +17,43 @@ module list
# echo "## MPI (one way, double the latency for round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 1"
sleep 1
echo "## Margo OFI/VERBS (round trip):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n "ofi+verbs;ofi_rxm://"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-latency -i 100000 -n "ofi+verbs;ofi_rxm://"
echo "## Margo OFI/VERBS (bw, 1MiB):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
echo "## Margo OFI/VERBS (bw, 8MiB):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
echo "## Margo OFI/VERBS (bw, 1MB unaligned):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20
sleep 1
echo "## Margo OFI/VERBS (round trip, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-latency -i 100000 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -t 0,0
echo "## Margo OFI/VERBS (bw, 1MiB, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 1 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 8 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 8 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 8MiB, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MB unaligned, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "ofi+verbs;ofi_rxm://" -c 8 -D 20 -t 0,0
# echo "## Mercury-runner CCI/VERBS (round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=verbs;ofi_rxm:// h1
# mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=verbs;ofi_rxm:// h1
......@@ -29,6 +29,8 @@ export LD_LIBRARY_PATH=$LIBRARY_PATH
# set up storage space
declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
echo "## testing launcher placement:"
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} hostname
......@@ -40,26 +42,26 @@ aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1
echo "## Bake OFI/GNI (8x concurrency):"
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8
echo "## Bake OFI/GNI (Hg busy spin):"
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 -t 0,0
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 -t 0,0
echo "## Bake OFI/GNI (8x concurrency, Hg busy spin):"
aprun -n 1 -N 1 -L ${nodes[0]} hostname
aprun -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 -t 0,0
aprun -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 -t 0,0
......@@ -22,6 +22,8 @@ module list
export LD_LIBRARY_PATH=$LIBRARY_PATH
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
#ldd ./margo-p2p-latency
# NOTE: needed as of January 2018 to avoid conflicts between MPI and
......@@ -37,40 +39,40 @@ export LD_LIBRARY_PATH=$LIBRARY_PATH
echo "## Margo OFI/GNI (round trip):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000
echo "## Margo OFI/GNI (bw, 1MiB):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 8MiB):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (round trip, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20-t 0,0
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20-t 0,0
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
# echo "## Margo BMI/TCP (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# echo "## Margo BMI/TCP (bw):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# echo "## Margo BMI/TCP (bw, 8x concurrency):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment