Commit cef48446 authored by Matthieu Dorier's avatar Matthieu Dorier

some modifications to workstation scripts

parent 97611636
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
#COBALT -A CSC250STDM12
#COBALT -q debug-cache-quad
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt
# necessary when using the udreg option in Mercury
export MPICH_GNI_NDREG_ENTRIES=1024
module swap PrgEnv-intel PrgEnv-gnu
module load cce
SANDBOX=$1
. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg
......
#!/bin/bash
#COBALT -n 2
#COBALT -t 20
#COBALT --mode script
#COBALT -A CSC250STDM12
#COBALT -q debug-cache-quad
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt
# necessary when using the udreg option in Mercury
export MPICH_GNI_NDREG_ENTRIES=1024
module swap PrgEnv-intel PrgEnv-gnu
module load cce
. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg
spack load -r bake
module list
export LD_LIBRARY_PATH=$LIBRARY_PATH
echo "### NOTE: all benchmarks are using numactl to keep processes on socket 0"
#ldd ./margo-p2p-latency
# NOTE: needed as of January 2018 to avoid conflicts between MPI and
# libfabric GNI provider
# NOTE: doing this with -e option to aprun
# NOTE: update as as of September 2018, this is no longer required now that
# mercury has switched back to alternative registration method for GNI
# export MPICH_GNI_NDREG_ENTRIES=2000
# echo "## MPI (one way, double the latency for round trip):"
# aprun -n 2 -N 1 ./osu_latency
echo "## Margo OFI/GNI (round trip):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000
echo "## Margo OFI/GNI (bw, 1MiB):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 8MiB):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20
echo "## Margo OFI/GNI (round trip, Hg busy spin):"
aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 8388608 -n ofi+gni://ipogif0:5000 -c 8 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 1 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
aprun -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1000000 -n ofi+gni://ipogif0:5000 -c 8 -D 20 -t 0,0
# echo "## Margo BMI/TCP (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./margo-p2p-latency -i 100000 -n bmi+tcp://
# echo "## Margo BMI/TCP (bw):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 1 -D 20
# echo "## Margo BMI/TCP (bw, 8x concurrency):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 numactl -N 0 -m 0 ./margo-p2p-bw -x 1048576 -n bmi+tcp:// -c 8 -D 20
# echo "## WARNING: Margo OFI/TCP (disabled because it hangs as of 8/10/2017)"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+tcp://ipogif0:5000
# figure out nodes and hostnames for use with fi_
# declare -a nodes=($(python /home/carns/bin/run_on_all_nids.py));
# server_hostname=$(printf "nid%05d\n" ${nodes[0]})
# echo "## fi_pingpong OFI/GNI (one way, double the usec/xfer for round trip):"
# aprun -n 1 -N 1 -L ${nodes[0]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 > $COBALT_JOBID.fi_pingpong_server.log &
# sleep 5
# aprun -n 1 -N 1 -L ${nodes[1]} -b -p carns-test ./fi_pingpong -p gni -S 1 -I 10000 $server_hostname > $COBALT_JOBID.fi_pingpong_client.log &
# wait
# cat $COBALT_JOBID.fi_pingpong_client.log
# one more margo run, this time with diagnostics enabled
# echo "## Margo OFI/GNI (round trip) with diagnostics:"
# aprun -n 2 -N 1 ./margo-p2p-latency -i 100000 -n ofi+gni://ipogif0:5000 -d $COBALT_JOBID.diag
# echo "## Mercury-runner OFI/GNI (round trip):"
# aprun -e MPICH_GNI_NDREG_ENTRIES=2000 -n 2 -N 1 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=ofi+gni://ipogif0:5000 h1
#!/bin/bash
SANDBOX=$1
. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg
spack load -r bake
module list
export LD_LIBRARY_PATH=$LIBRARY_PATH
#ldd ./margo-p2p-latency
# NOTE: needed as of January 2018 to avoid conflicts between MPI and
# libfabric GNI provider
# NOTE: doing this with -e option to aprun
# NOTE: update as as of September 2018, this is no longer required now that
# mercury has switched back to alternative registration method for GNI
# export MPICH_GNI_NDREG_ENTRIES=2000
# echo "## MPI (one way, double the latency for round trip):"
# aprun -n 2 -N 1 ./osu_latency
echo "## Margo OFI/GNI (round trip):"
mpirun -np 2 ./margo-p2p-latency -i 100000 -n tcp
echo "## Margo OFI/GNI (bw, 1MiB):"
mpirun -np 2 ./margo-p2p-bw -x 1048576 -n tcp -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency):"
mpirun -np 2 ./margo-p2p-bw -x 1048576 -n tcp -c 8 -D 20
echo "## Margo OFI/GNI (bw, 8MiB):"
mpirun -np 2 ./margo-p2p-bw -x 8388608 -n tcp -c 1 -D 20
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency):"
mpirun -np 2 ./margo-p2p-bw -x 8388608 -n tcp -c 8 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned):"
mpirun -np 2 ./margo-p2p-bw -x 1000000 -n tcp -c 1 -D 20
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency):"
mpirun -np 2 ./margo-p2p-bw -x 1000000 -n tcp -c 8 -D 20
echo "## Margo OFI/GNI (round trip, Hg busy spin):"
mpirun -np 2 ./margo-p2p-latency -i 100000 -n tcp -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, Hg busy spin):"
mpirun -np 2 ./margo-p2p-bw -x 1048576 -n tcp -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 1MiB, 8x concurrency, Hg busy spin):"
mpirun -np 2 ./margo-p2p-bw -x 1048576 -n tcp -c 8 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, Hg busy spin):"
mpirun -np 2 ./margo-p2p-bw -x 8388608 -n tcp -c 1 -D 20 -t 0,0
echo "## Margo OFI/GNI (bw, 8MiB, 8x concurrency, Hg busy spin):"
mpirun -np 2 ./margo-p2p-bw -x 8388608 -n tcp -c 8 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, Hg busy spin):"
mpirun -np 2 ./margo-p2p-bw -x 1000000 -n tcp -c 1 -D 20-t 0,0
echo "## Margo OFI/GNI (bw, 1MB unaligned, 8x concurrency, Hg busy spin):"
mpirun -np 2 ./margo-p2p-bw -x 1000000 -n tcp -c 8 -D 20 -t 0,0
#!/bin/bash
#COBALT -n 1
#COBALT -t 20
#COBALT --mode script
#COBALT -A CSC250STDM12
#COBALT -q debug-cache-quad
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt
# necessary when using the udreg option in Mercury
export MPICH_GNI_NDREG_ENTRIES=1024
module swap PrgEnv-intel PrgEnv-gnu
module load cce
SANDBOX=$1
. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg
......
......@@ -9,8 +9,6 @@ set -e
# default to O3 optimizations unless otherwise specified
export CFLAGS="-O3"
# dynamic link everything by default
export CRAYPE_LINK_TYPE=dynamic
# location of this script
ORIGIN=$PWD
......@@ -42,7 +40,7 @@ echo "=== BUILD SPACK PACKAGES AND LOAD ==="
spack compiler find
spack compilers
# use our own packages.yaml for theta-specific preferences
# use our own packages.yaml for the workstation-specific preferences
cp $ORIGIN/packages.yaml $SPACK_ROOT/etc/spack
# add external repo for mochi. Note that this will not modify the
# user's ~/.spack/ files because we modified $HOME above
......@@ -52,7 +50,7 @@ spack repo list
# clean out any stray packages from previous runs, just in case
spack uninstall -R -y argobots mercury libfabric || true
# ior acts as our "apex" package here, causing several other packages to build
spack install ior@develop +mobject
spack install ior@develop +mobject ^bake@develop
# deliberately repeat setup-env step after building modules to ensure
# that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh
......@@ -80,18 +78,15 @@ cp $PREFIX/bin/bake-p2p-bw $JOBDIR
cp $PREFIX/bin/pmdk-bw $JOBDIR
cd $JOBDIR
JOBID=`qsub --env SANDBOX=$SANDBOX ./margo-regression.qsub`
cqwait $JOBID
JOBID2=`qsub --env SANDBOX=$SANDBOX ./bake-regression.qsub`
cqwait $JOBID2
JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
cqwait $JOBID3
./margo-regression.sh $SANDBOX
#./bake-regression.sh $SANDBOX
#./pmdk-regression.sh $SANDBOX
echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
# gather output, strip out funny characters, mail
cat $JOBID.* $JOBID2.* $JOBID3.* > combined.$JOBID.txt
#dos2unix combined.$JOBID.txt
mailx -r carns@mcs.anl.gov -s "mochi-regression (theta)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
mailx -r mdorier@anl.gov -s "mochi-regression (MCS workstation)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
cat combined.$JOBID.txt
cd /tmp
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment