Skip to content

GitLab

Commit 08779fb4 authored by Philip Carns's avatar Philip Carns
Browse files

switch to aprun -cc none to avoid cpu affinity 

- seems like a somewhat more general alternative to -d 64, but with the
  same net result
parent 759d644f
Hide whitespace changes
Inline Side-by-side
Showing with 70 additions and 70 deletions
perf-regression/theta/bake-regression.qsub
View file @ 08779fb4
......@@ -30,36 +30,36 @@ export LD_LIBRARY_PATH=$LIBRARY_PATH
# set up storage space
declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
echo "### NOTE: all benchmarks are using aprun -d 64 to allow processes to run on all available cores; the default aprun settings limit processes to one core and produce poor performance because of contention between internal threads"
echo "### NOTE: all benchmarks are using aprun -cc none to allow processes to run on all available cores; the default aprun settings limit processes to one core and produce poor performance because of contention between internal threads"
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
echo "## testing launcher placement:"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
echo "## Bake OFI/GNI:"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -d 64 -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 1
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -cc none -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 1
echo "## Bake OFI/GNI (8x concurrency):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -d 64 -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 8
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -cc none -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 8
echo "## Bake OFI/GNI (Hg busy spin):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -d 64 -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 1 -t 0,0
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -cc none -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 1 -t 0,0
echo "## Bake OFI/GNI (8x concurrency, Hg busy spin):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -d 64 -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 8 -t 0,0
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} bake-mkpool -s 60G /dev/shm/foo.dat
aprun -cc none -n 2 -N 1 -L ${nodes[0]},${nodes[1]} numactl -N 0 -m 0 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "gni://" -p /dev/shm/foo.dat -c 8 -t 0,0
perf-regression/theta/margo-regression.qsub
View file @ 08779fb4
......@@ -32,45 +32,45 @@ echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
# aprun -n 2 -N 1 ./osu_latency
echo "## Margo OFI/GNI (round trip):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-latency -i 100000 -n "gni://"
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-latency -i 100000 -n "gni://"
echo "## Margo OFI/GNI (bw, 1MiB):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 1 -D 20
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 8 -D 20
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 8 -D 20
echo "## Margo OFI/GNI (bw, 8MiB):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 1 -D 20
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 1 -D 20
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 8 -D 20
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 8 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 1 -D 20
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 8 -D 20
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 8 -D 20
echo "## Margo OFI/GNI (round trip, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-latency -i 100000 -n "gni://" -t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-latency -i 100000 -n "gni://" -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 1 -D 20 -t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 8 -D 20 -t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n "gni://" -c 8 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 1 -D 20 -t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 8 -D 20-t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n "gni://" -c 8 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 1 -D 20-t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 1 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 8 -D 20 -t 0,0
aprun -cc none -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c 8 -D 20 -t 0,0
# echo "## Margo BMI/TCP (round trip):"
# aprun -d 64 -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# aprun -cc none -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# echo "## Margo BMI/TCP (bw):"
# aprun -d 64 -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# aprun -cc none -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# echo "## Margo BMI/TCP (bw, 8x concurrency):"
# aprun -d 64 -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# aprun -cc none -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -d 64 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000
# aprun -cc none -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000
# figure out nodes and hostnames for use with fi_
......@@ -78,17 +78,17 @@ aprun -d 64 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n "gni://" -c
# server_hostname=$(printf "nid%05d\n" ${nodes[0]})
# echo "## fi_pingpong OFI/GNI (one way, double the usec/xfer for round trip):"
# aprun -d 64 -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log &
# aprun -cc none -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log &
# sleep 5
# aprun -d 64 -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log &
# aprun -cc none -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log &
# wait
# cat $COBALT_JOBID.fi_pingpong_client.log
# one more margo run, this time with diagnostics enabled
# echo "## Margo OFI/GNI (round trip) with diagnostics:"
# aprun -d 64 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n "gni://" -d $COBALT_JOBID.diag
# aprun -cc none -n 2 -N 1 ./margo-p2p-latency -i 100000 -n "gni://" -d $COBALT_JOBID.diag
# echo "## Mercury-runner OFI/GNI (round trip):"
# aprun -d 64 -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0="gni://" h1
# aprun -cc none -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0="gni://" h1
perf-regression/theta/pmdk-regression.qsub
View file @ 08779fb4
......@@ -30,37 +30,37 @@ export LD_LIBRARY_PATH=$LIBRARY_PATH
# set up storage space
declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
echo "### NOTE: all benchmarks are using aprun -d 64 to allow processes to run on all available cores; the default aprun settings limit processes to one core and produce poor performance because of contention between internal threads"
echo "### NOTE: all benchmarks are using aprun -cc none to allow processes to run on all available cores; the default aprun settings limit processes to one core and produce poor performance because of contention between internal threads"
echo "## PMDK (8x concurrency):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} truncate -s 60G /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} truncate -s 60G /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8
echo "## PMDK (8x concurrency, 8 es):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} truncate -s 60G /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8 -T 8
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} truncate -s 60G /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8 -T 8
echo "## PMDK (8x concurrency, preallocated pool):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} dd if=/dev/zero of=/dev/shm/foo.dat bs=1M count=61440
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} dd if=/dev/zero of=/dev/shm/foo.dat bs=1M count=61440
aprun -cc none -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8
echo "## PMDK (8x concurrency, 8 es, preallocated pool):"
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} hostname
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} dd if=/dev/zero of=/dev/shm/foo.dat bs=1M count=61440
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -d 64 -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8 -T 8
aprun -cc none -n 1 -N 1 -L ${nodes[0]} hostname
aprun -cc none -n 1 -N 1 -L ${nodes[0]} rm -f /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} dd if=/dev/zero of=/dev/shm/foo.dat bs=1M count=61440
aprun -cc none -n 1 -N 1 -L ${nodes[0]} pmempool create obj /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ls -alh /dev/shm/foo.dat
aprun -cc none -n 1 -N 1 -L ${nodes[0]} ./pmdk-bw -x 16777216 -m 34359738368 -p /dev/shm/foo.dat -c 8 -T 8
perf-regression/theta/separate-ssg.qsub
View file @ 08779fb4
......@@ -27,7 +27,7 @@ apmgr pdomain -c -u $SSG_PDOMAIN
SSG_STATEFILE=ssg-state.cfg
rm -f $SSG_STATEFILE
aprun -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-create ${SSG_STATEFILE} &
aprun -cc none -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-create ${SSG_STATEFILE} &
sleep_time=1;
while [ ! -e $SSG_STATEFILE ] ; do
......@@ -37,7 +37,7 @@ done
xxd $SSG_STATEFILE
aprun -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-attach ofi+gni ${SSG_STATEFILE}
aprun -cc none -p ${SSG_PDOMAIN} -n 1 -N 1 ./ssg-test-separate-group-attach ofi+gni ${SSG_STATEFILE}
kill %1
apmgr pdomain -r -u ${SSG_PDOMAIN}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment