Commit ffe82e8a authored by Philip Carns's avatar Philip Carns

fix jumbled files

parent 4a072bed
...@@ -8,14 +8,14 @@ ...@@ -8,14 +8,14 @@
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now # XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt # module unload xalt
get_mercury_lib_path() {
module show `spack module tcl find mercury` |&grep LIBRARY_PATH | awk '{print $3}'
}
get_pmdk_lib_path() { get_pmdk_lib_path() {
module show `spack module tcl find pmem` |&grep LIBRARY_PATH | awk '{print $3}' module show `spack module tcl find pmem` |&grep LIBRARY_PATH | awk '{print $3}'
} }
get_mercury_lib_path() {
module show `spack module tcl find mercury` |&grep LIBRARY_PATH | awk '{print $3}'
}
get_remi_lib_path() { get_remi_lib_path() {
module show `spack module tcl find remi` |&grep LIBRARY_PATH | awk '{print $3}' module show `spack module tcl find remi` |&grep LIBRARY_PATH | awk '{print $3}'
} }
...@@ -34,82 +34,31 @@ module list ...@@ -34,82 +34,31 @@ module list
# test programs. Not sure why. This hack manually adds the correct path to LD_LIBRARY_PATH. # test programs. Not sure why. This hack manually adds the correct path to LD_LIBRARY_PATH.
LIB_PATH_HACK=$(get_mercury_lib_path) LIB_PATH_HACK=$(get_mercury_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH" export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
# ditto for pmdk and apparently? # ditto for pmdk apparently
LIB_PATH_HACK=$(get_pmdk_lib_path) LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH" export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH
LIB_PATH_HACK=$(get_remi_lib_path) LIB_PATH_HACK=$(get_remi_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH" export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH
#ldd ./margo-p2p-latency echo "## Bake OFI/GNI:"
rm -f /dev/shm/foo.dat
# NOTE: needed as of January 2018 to avoid conflicts between MPI and bake-mkpool -s 60G /dev/shm/foo.dat
# libfabric GNI provider aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1
# NOTE: doing this with -e option to aprun
# NOTE: update as as of September 2018, this is no longer required now that
# mercury has switched back to alternative registration method for GNI
# export MPICH_GNI_NDREG_ENTRIES=2000
# echo "## MPI (one way, double the latency for round trip):"
# aprun -n 2 -N 1 ./osu_latency
echo "## Margo OFI/GNI (round trip):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000
echo "## Margo OFI/GNI (bw, 1MiB):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 8MiB):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (round trip, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
# echo "## Margo BMI/TCP (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# echo "## Margo BMI/TCP (bw):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# echo "## Margo BMI/TCP (bw, 8x concurrency):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000
# figure out nodes and hostnames for use with fi_
# declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
# server_hostname=$(printf "nid%05d\n" ${nodes[0]})
# echo "## fi_pingpong OFI/GNI (one way, double the usec/xfer for round trip):" echo "## Bake OFI/GNI (8x concurrency):"
# aprun -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log & rm -f /dev/shm/foo.dat
# sleep 5 bake-mkpool -s 60G /dev/shm/foo.dat
# aprun -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log & aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8
# wait echo "## Bake OFI/GNI (Hg busy spin):"
# cat $COBALT_JOBID.fi_pingpong_client.log rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 -t 0,0
# one more margo run, this time with diagnostics enabled echo "## Bake OFI/GNI (8x concurrency, Hg busy spin):"
# echo "## Margo OFI/GNI (round trip) with diagnostics:" rm -f /dev/shm/foo.dat
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -d $COBALT_JOBID.diag bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 -t 0,0
# echo "## Mercury-runner OFI/GNI (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=ofi+gni://ipogif0:5000 h1
...@@ -8,14 +8,19 @@ ...@@ -8,14 +8,19 @@
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now # XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt # module unload xalt
get_mercury_lib_path() {
module show `spack module tcl find mercury` |&grep LIBRARY_PATH | awk '{print $3}'
}
get_pmdk_lib_path() { get_pmdk_lib_path() {
module show `spack module tcl find pmem` |&grep LIBRARY_PATH | awk '{print $3}' module show `spack module tcl find pmem` |&grep LIBRARY_PATH | awk '{print $3}'
} }
get_mercury_lib_path() { get_remi_lib_path() {
module show `spack module tcl find mercury` |&grep LIBRARY_PATH | awk '{print $3}' module show `spack module tcl find remi` |&grep LIBRARY_PATH | awk '{print $3}'
} }
module swap PrgEnv-intel PrgEnv-gnu module swap PrgEnv-intel PrgEnv-gnu
module load cce module load cce
...@@ -29,28 +34,82 @@ module list ...@@ -29,28 +34,82 @@ module list
# test programs. Not sure why. This hack manually adds the correct path to LD_LIBRARY_PATH. # test programs. Not sure why. This hack manually adds the correct path to LD_LIBRARY_PATH.
LIB_PATH_HACK=$(get_mercury_lib_path) LIB_PATH_HACK=$(get_mercury_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH" export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
# ditto for pmdk apparently # ditto for pmdk and apparently?
LIB_PATH_HACK=$(get_pmdk_lib_path) LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH" export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
LIB_PATH_HACK=$(get_remi_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH echo LD_LIBRARY_PATH: $LD_LIBRARY_PATH
echo "## Margo OFI/GNI:" #ldd ./margo-p2p-latency
rm -f /dev/shm/foo.dat
bake-mkpool -s 60G /dev/shm/foo.dat # NOTE: needed as of January 2018 to avoid conflicts between MPI and
aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 # libfabric GNI provider
# NOTE: doing this with -e option to aprun
# NOTE: update as as of September 2018, this is no longer required now that
# mercury has switched back to alternative registration method for GNI
# export MPICH_GNI_NDREG_ENTRIES=2000
# echo "## MPI (one way, double the latency for round trip):"
# aprun -n 2 -N 1 ./osu_latency
echo "## Margo OFI/GNI (round trip):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000
echo "## Margo OFI/GNI (bw, 1MiB):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 8MiB):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (round trip, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
# echo "## Margo BMI/TCP (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# echo "## Margo BMI/TCP (bw):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# echo "## Margo BMI/TCP (bw, 8x concurrency):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000
# figure out nodes and hostnames for use with fi_
# declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
# server_hostname=$(printf "nid%05d\n" ${nodes[0]})
echo "## Margo OFI/GNI (8x concurrency):" # echo "## fi_pingpong OFI/GNI (one way, double the usec/xfer for round trip):"
rm -f /dev/shm/foo.dat # aprun -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log &
bake-mkpool -s 60G /dev/shm/foo.dat # sleep 5
aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 # aprun -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log &
echo "## Margo OFI/GNI (Hg busy spin):" # wait
rm -f /dev/shm/foo.dat # cat $COBALT_JOBID.fi_pingpong_client.log
bake-mkpool -s 60G /dev/shm/foo.dat
aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 1 -t 0,0
echo "## Margo OFI/GNI (8x concurrency, Hg busy spin):" # one more margo run, this time with diagnostics enabled
rm -f /dev/shm/foo.dat # echo "## Margo OFI/GNI (round trip) with diagnostics:"
bake-mkpool -s 60G /dev/shm/foo.dat # aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -d $COBALT_JOBID.diag
aprun -n 2 -N 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "ofi+gni://ipogif0:5000" -p /dev/shm/foo.dat -c 8 -t 0,0
# echo "## Mercury-runner OFI/GNI (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=ofi+gni://ipogif0:5000 h1
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment