Commit b3126cf6 authored by Philip Carns's avatar Philip Carns

explicitly specify mlx5_0 device on cooley

parent 9d52fd38
......@@ -33,7 +33,7 @@ echo "### verbs kove pmdk no threads"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p $PMDK_PATH -c 12
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
......@@ -42,7 +42,7 @@ echo "### verbs kove pmdk 4 rpc threads"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p $PMDK_PATH -c 12 -r 4
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
......@@ -51,7 +51,7 @@ echo "### verbs kove pmdk no threads, pipelined"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p $PMDK_PATH -c 12 -i
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
......@@ -60,7 +60,7 @@ echo "### verbs kove pmdk 4 rpc threads, pipelined"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p $PMDK_PATH -c 12 -r 4 -i
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
......@@ -76,7 +76,7 @@ sleep 1
dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdha oflag=direct
mount /kovefs/xfsa
bake-mkpool /kovefs/xfsa/bake.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfsa/bake.dat -c 12 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /kovefs/xfsa/bake.dat -c 12 -i
umount /kovefs/xfsa
kdsa_manage disconnect xpdha
......@@ -87,7 +87,7 @@ sleep 1
dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdha oflag=direct
mount /kovefs/xfsa
bake-mkpool /kovefs/xfsa/bake.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfsa/bake.dat -c 12 -r 4 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /kovefs/xfsa/bake.dat -c 12 -r 4 -i
umount /kovefs/xfsa
kdsa_manage disconnect xpdha
......@@ -23,30 +23,30 @@ echo "### NOTE: all benchmarks are using numactl to keep processes on socket 1"
echo "## Bake OFI/VERBS:"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /dev/shm/foo.dat -c 1
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /dev/shm/foo.dat -c 1
echo "## Bake OFI/VERBS (8x concurrency):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /dev/shm/foo.dat -c 8
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /dev/shm/foo.dat -c 8
echo "## Bake OFI/VERBS (4 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /dev/shm/foo.dat -c 1 -r 4
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /dev/shm/foo.dat -c 1 -r 4
echo "## Bake OFI/VERBS (8x concurrency, 4 rpc es):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /dev/shm/foo.dat -c 8 -r 4
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /dev/shm/foo.dat -c 8 -r 4
echo "## Bake OFI/VERBS (4 rpc es, pipelining):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /dev/shm/foo.dat -c 1 -r 4 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /dev/shm/foo.dat -c 1 -r 4 -i
echo "## Bake OFI/VERBS (8x concurrency, 4 rpc es, pipelining):"
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /dev/shm/foo.dat -c 8 -r 4 -i
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://mlx5_0" -p /dev/shm/foo.dat -c 8 -r 4 -i
......@@ -26,19 +26,19 @@ echo "### NOTE: all benchmarks are using numactl to keep processes on socket 1"
sleep 1
echo "## Margo OFI/VERBS (round trip):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-latency -i 100000 -n "verbs://"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-latency -i 100000 -n "verbs://mlx5_0"
echo "## Margo OFI/VERBS (bw, 1MiB):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "verbs://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "verbs://mlx5_0" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "verbs://" -c 8 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "verbs://mlx5_0" -c 8 -D 20
echo "## Margo OFI/VERBS (bw, 8MiB):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://mlx5_0" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 8 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://mlx5_0" -c 8 -D 20
echo "## Margo OFI/VERBS (bw, 1MB unaligned):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://mlx5_0" -c 1 -D 20
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 8 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://mlx5_0" -c 8 -D 20
sleep 1
......@@ -52,13 +52,13 @@ mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "
echo "## Margo OFI/VERBS (bw, 1MiB, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1048576 -n "ofi+verbs;ofi_rxm://mlx5_0:3339" -c 8 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 8MiB, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 1 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://mlx5_0" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 8MiB, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://" -c 8 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 8388608 -n "verbs://mlx5_0" -c 8 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MB unaligned, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 1 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://mlx5_0" -c 1 -D 20 -t 0,0
echo "## Margo OFI/VERBS (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://" -c 8 -D 20 -t 0,0
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-bw -x 1000000 -n "verbs://mlx5_0" -c 8 -D 20 -t 0,0
......
......@@ -21,11 +21,11 @@ spack load -r pmdk
spack find --loaded
echo "## Margo OFI/VERBS (vector benchmark with len 1, 512KiB xfers):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-vector -x 524288 -n "verbs://" -c 1 -D 20
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-vector -x 524288 -n "verbs://mlx5_0" -c 1 -D 20
sleep 1
echo "## Margo OFI/VERBS (vector benchmark with len 256, 512KiB xfers):"
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-vector -x 524288 -n "verbs://" -c 1 -D 20 -v 256
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./margo-p2p-vector -x 524288 -n "verbs://mlx5_0" -c 1 -D 20 -v 256
......@@ -21,7 +21,7 @@ tail -n $((nnodes/2)) $COBALT_NODEFILE > client_nodes
echo " ==== running server "
export MOBJECT_CLUSTER_FILE=mobject-cluster.gid
mpirun -f $COBALT_NODEFILE -n 1 mobject-server-daemon "verbs://" $MOBJECT_CLUSTER_FILE &
mpirun -f $COBALT_NODEFILE -n 1 mobject-server-daemon "verbs://mlx5_0" $MOBJECT_CLUSTER_FILE &
# give the mobject-server nodes a chance to set up
sleep 5
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment