Commit 2c967838 authored by Philip Carns's avatar Philip Carns

Merge branch 'carns/cooley-bake-kove'

parents 50aaf9b3 a5fdbc02
#!/bin/bash
#COBALT -n 2
#COBALT -t 30
#COBALT --mode script
#COBALT -A radix-io
#COBALT -q ibleaf3-debug
# This job tests bake performance on top of a kove volume, alternatively with either
# pmdk or the file backend to bake. Note that it is hard coded to use a specific
# xpd and volume; this job assumes it has exclusive access.
export FI_FORK_UNSAFE=1
export HOME=$SANDBOX
export KDSA_MPIO_MAX_CONN=1
export PMDK_PATH="/xmem/xpdha"
. $SANDBOX/spack/share/spack/setup-env.sh
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
spack load -r ssg
spack load -r $BAKE_STABLE_VER
spack load -r mvapich2
module list
echo "### NOTE: all results in this output are using Kove XPD volumes"
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11, also using numactl to constrain memory allocations"
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
echo "### verbs kove pmdk no threads"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
echo "### verbs kove pmdk 4 rpc threads"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
echo "### verbs kove pmdk no threads, pipelined"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -i
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
echo "### verbs kove pmdk 4 rpc threads, pipelined"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4 -i
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
# reset to use file backend. Note that we use a different name for the
# benchmark executable as well for the one that was linked against this
# backend and has it's rpath set accordingly.
spack unload $BAKE_STABLE_VER
spack load bake@dev-file-backend
echo "### verbs kove filesystem no threads, pipelined"
sleep 1
dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdha oflag=direct
mount /kovefs/xfsa
bake-mkpool /kovefs/xfsa/bake.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfsa/bake.dat -c 12 -i
umount /kovefs/xfsa
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
echo "### verbs kove filesystem 4 rpc threads, pipelined"
sleep 1
dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdha oflag=direct
mount /kovefs/xfsa
bake-mkpool /kovefs/xfsa/bake.dat
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfsa/bake.dat -c 12 -r 4 -i
umount /kovefs/xfsa
kdsa_manage disconnect xpdha
......@@ -9,8 +9,9 @@ export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
spack load -r ssg
spack load -r bake
spack load -r $BAKE_STABLE_VER
spack load -r mvapich2
module list
......
......@@ -9,8 +9,10 @@ export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
spack load -r ssg
spack load -r bake
spack load -r $BAKE_STABLE_VER
spack load -r mvapich2
module list
......
......@@ -6,11 +6,12 @@
#COBALT -q ibleaf3-debug
export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
spack load -r ssg
spack load -r bake
spack load -r $BAKE_STABLE_VER
spack load -r mvapich2
module list
......
......@@ -33,6 +33,7 @@ cp $ORIGIN/margo-regression.qsub $JOBDIR
cp $ORIGIN/bake-regression.qsub $JOBDIR
cp $ORIGIN/pmdk-regression.qsub $JOBDIR
cp $ORIGIN/mobject-regression.qsub $JOBDIR
cp $ORIGIN/bake-kove.qsub $JOBDIR
# set up build environment
cd $SANDBOX
......@@ -58,13 +59,17 @@ spack bootstrap
spack uninstall -R -y argobots mercury rdma-core libfabric || true
# ior acts as our "apex" package here, causing several other packages to build
spack install ior@develop +mobject
# check what stable version of bake we got
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
# load an additional version of bake that uses a file backend
spack install bake@dev-file-backend
# deliberately repeat setup-env step after building modules to ensure
# that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh
# load ssg and bake because they are needed by things compiled outside of
# spack later in this script
spack load -r ssg
spack load -r bake
spack load -r $BAKE_STABLE_VER
# sds-tests
echo "=== BUILDING SDS TEST PROGRAMS ==="
......@@ -77,6 +82,17 @@ cd build
make -j 3
make install
# switch bake versions and build another copy with file backend
echo "=== BUILDING SDS TEST PROGRAMS WITH FILE BACKEND ==="
spack unload $BAKE_STABLE_VER
spack load bake@dev-file-backend
cd $SANDBOX/sds-tests
mkdir build-file
cd build-file
../configure --prefix=${PREFIX}-file CC=mpicc
make -j 3
make install
# mercury-runner benchmark
# echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
# cd $SANDBOX/mercury-runner
......@@ -91,6 +107,7 @@ echo "=== SUBMITTING AND WAITING FOR JOB ==="
cp $PREFIX/bin/margo-p2p-latency $JOBDIR
cp $PREFIX/bin/margo-p2p-bw $JOBDIR
cp $PREFIX/bin/bake-p2p-bw $JOBDIR
cp ${PREFIX}-file/bin/bake-p2p-bw $JOBDIR/bake-p2p-bw-file
cp $PREFIX/bin/pmdk-bw $JOBDIR
# cp $PREFIX/bin/mercury-runner $JOBDIR
cd $JOBDIR
......@@ -104,10 +121,12 @@ JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
cqwait $JOBID3
JOBID4=`qsub --env SANDBOX=$SANDBOX ./mobject-regression.qsub`
cqwait $JOBID4
JOBID5=`qsub --env SANDBOX=$SANDBOX ./bake-kove.qsub`
cqwait $JOBID5
echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
# gather output, strip out funny characters, mail
cat $JOBID.* $JOBID2.* $JOBID3.* $JOBID4.*> combined.$JOBID.txt
cat $JOBID.* $JOBID2.* $JOBID3.* $JOBID4.* $JOBID5.* > combined.$JOBID.txt
dos2unix combined.$JOBID.txt
mailx -s "mochi-regression (cooley)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment