add kove regression benchmarks

- includes two bake builds: one with pmdk, one with file backend
- actual runs do not work yet due to a mount problem

perf-regression/cooley/bake-kove.qsub

+#!/bin/bash
+#COBALT -n 2
+#COBALT -t 30
+#COBALT --mode script
+#COBALT -A radix-io
+#COBALT -q ibleaf3-debug
+
+# This job tests bake performance on top of a kove volume, alternatively with either 
+# pmdk or the file backend to bake.  Note that it is hard coded to use a specific
+# xpd and volume; this job assumes it has exclusive access.
+
+export FI_FORK_UNSAFE=1
+export HOME=$SANDBOX
+export KDSA_MPIO_MAX_CONN=1
+export PMDK_PATH="/xmem/xpdgb"
+
+. $SANDBOX/spack/share/spack/setup-env.sh
+BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
+spack load -r ssg 
+spack load -r $BAKE_STABLE_VER
+spack load -r mvapich2
+
+module list
+
+echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11, also using numactl to constrain memory allocations"
+
+kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdgb
+
+echo "### verbs kove pmdk no threads"
+sleep 1
+dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
+bake-mkpool $PMDK_PATH
+mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 
+
+kdsa_manage disconnect xpdgb
+kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdgb
+
+echo "### verbs kove pmdk 4 rpc threads"
+sleep 1
+dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
+bake-mkpool $PMDK_PATH
+mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4
+
+kdsa_manage disconnect xpdgb
+kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdgb
+
+echo "### verbs kove pmdk no threads, pipelined"
+sleep 1
+dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
+bake-mkpool $PMDK_PATH
+mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -i
+
+kdsa_manage disconnect xpdgb
+kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdgb
+
+echo "### verbs kove pmdk 4 rpc threads, pipelined"
+sleep 1
+dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
+bake-mkpool $PMDK_PATH
+mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4 -i
+
+kdsa_manage disconnect xpdgb
+kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdgb
+
+# reset to use file backend
+
+spack unload $BAKE_STABLE_VER
+spack load bake@dev-file-backend
+
+echo "### verbs kove filesystem no threads, pipelined"
+sleep 1
+dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdgb oflag=direct
+mount /kovefs/xfs
+bake-mkpool /kovefs/xfs/bake.dat
+mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfs/bake.dat -c 12 -i
+umount /kovefs/xfs
+
+kdsa_manage disconnect xpdgb
+kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdgb
+
+echo "### verbs kove filesystem 4 rpc threads, pipelined"
+sleep 1
+dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdgb oflag=direct
+mount /kovefs/xfs
+bake-mkpool /kovefs/xfs/bake.dat
+mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfs/bake.dat -c 12 -r 4 -i
+umount /kovefs/xfs
+
+kdsa_manage disconnect xpdgb

perf-regression/cooley/bake-regression.qsub

@@ -9,8 +9,9 @@ export HOME=$SANDBOX
 export FI_FORK_UNSAFE=1
 
 . $SANDBOX/spack/share/spack/setup-env.sh
+BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
 spack load -r ssg 
-spack load -r bake
+spack load -r $BAKE_STABLE_VER
 spack load -r mvapich2
 
 module list

perf-regression/cooley/margo-regression.qsub

@@ -9,8 +9,10 @@ export HOME=$SANDBOX
 export FI_FORK_UNSAFE=1
 
 . $SANDBOX/spack/share/spack/setup-env.sh
+BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
 spack load -r ssg 
-spack load -r bake
+spack load -r $BAKE_STABLE_VER
+spack load -r mvapich2
 
 module list
 

perf-regression/cooley/pmdk-regression.qsub

@@ -6,11 +6,12 @@
 #COBALT -q ibleaf3-debug
 
 export HOME=$SANDBOX
-export FI_FORK_UNSAFE=1
 
 . $SANDBOX/spack/share/spack/setup-env.sh
+BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
 spack load -r ssg 
-spack load -r bake
+spack load -r $BAKE_STABLE_VER
+spack load -r mvapich2
 
 module list
 

perf-regression/cooley/run-regression.sh

@@ -33,6 +33,7 @@ cp $ORIGIN/margo-regression.qsub $JOBDIR
 cp $ORIGIN/bake-regression.qsub $JOBDIR
 cp $ORIGIN/pmdk-regression.qsub $JOBDIR
 cp $ORIGIN/mobject-regression.qsub $JOBDIR
+cp $ORIGIN/bake-kove.qsub $JOBDIR
 
 # set up build environment
 cd $SANDBOX
@@ -58,13 +59,17 @@ spack bootstrap
 spack uninstall -R -y argobots mercury rdma-core libfabric || true
 # ior acts as our "apex" package here, causing several other packages to build
 spack install ior@develop +mobject
+# check what stable version of bake we got
+BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
+# load an additional version of bake that uses a file backend
+spack install bake@dev-file-backend
 # deliberately repeat setup-env step after building modules to ensure
 #   that we pick up the right module paths
 . $SANDBOX/spack/share/spack/setup-env.sh
 # load ssg and bake because they are needed by things compiled outside of
 # spack later in this script
 spack load -r ssg
-spack load -r bake
+spack load -r $BAKE_STABLE_VER
 
 # sds-tests
 echo "=== BUILDING SDS TEST PROGRAMS ==="
@@ -104,10 +109,12 @@ JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
 cqwait $JOBID3
 JOBID4=`qsub --env SANDBOX=$SANDBOX ./mobject-regression.qsub`
 cqwait $JOBID4
+JOBID5=`qsub --env SANDBOX=$SANDBOX ./bake-kove.qsub`
+cqwait $JOBID5
 
 echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
 # gather output, strip out funny characters, mail
-cat $JOBID.* $JOBID2.* $JOBID3.* $JOBID4.*> combined.$JOBID.txt
+cat $JOBID.* $JOBID2.* $JOBID3.* $JOBID4.* $JOBID5.* > combined.$JOBID.txt
 dos2unix combined.$JOBID.txt
 mailx -s "mochi-regression (cooley)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt