revamp cooley regression scripts

- set $HOME to avoid perturbing ~/ config files for the user running this script
- remove deprecated --dirty and LD_PRELOAD hacks
- remove explicit set of SPACK_ROOT in favor of variable set by spack itself
- other misc streamlining improvements

perf-regression/cooley/README.spack.md

-To compile the full stack on cooley as of June 2018 using spack:
+To compile the full stack on cooley as of May 2019 using spack:
 
-# set up .soft.cooley like this:
-```
-+gcc-7.1.0
-+autotools-feb2016
-+cmake-3.9.1
-+mvapich2
-@default
-```
-
-# get spack repos:
+1. set up softenv by following the "soft add" commands in run-regression.sh, or add them to your .soft.cooley file
+2. get spack repos:
 ```
 git clone https://github.com/spack/spack.git
-# as of 2018-06-12, need to pull in PR changes to allow SPACK_SHELL override
+git clone git@xgitlab.cels.anl.gov:sds/sds-repo
 cd spack
-git remote add jrood-nrel https://github.com/jrood-nrel/spack.git
-git fetch --all
-git merge jrood-nrel/fix_spack_shell_bootstrap
 . share/spack/setup-env.sh
 spack bootstrap
-git clone git@xgitlab.cels.anl.gov:sds/sds-repo
+cd ../sds-repo
 spack repo add .
 ```
-
-This directory contains a packages.yaml that should be copied to one of the spack
-configuration locations documented here: https://spack.readthedocs.io/en/stable/configuration.html .  This could be your ~/.spack/linux/ directory, for example.
-The automation scripts in this directory use $SPACK_ROOT/etc/spack to avoid
-perturbing the account on which the regression tests are executed.
-
-```
-# compile everything and load module for ssg
-# note that the --dirty option is needed because gcc 7.1 on cooley only works if it can inherit LD_LIBRARY_PATH from softenv
-spack install --dirty ssg
-spack load -r ssg
-```
+3. copy packages.yaml from this directory to your personal configuration location (usually ~/.spack/linux/)
+4. compile and load the packages you want with `spack install` and `spack load`

perf-regression/cooley/bake-regression.qsub

@@ -5,10 +5,7 @@
 #COBALT -A radix-io
 #COBALT -q ibleaf3-debug
 
-get_pmdk_lib_path() {
-        module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
-}
-
+export HOME=$SANDBOX
 export FI_FORK_UNSAFE=1
 
 . $SANDBOX/spack/share/spack/setup-env.sh
@@ -18,10 +15,6 @@ spack load -r mvapich2
 
 module list
 
-# note, pmdk library not setting rpath correctly
-LIB_PATH_HACK=$(get_pmdk_lib_path)
-export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
-
 echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11"
 
 echo "## Bake OFI/VERBS:"

perf-regression/cooley/margo-regression.qsub

@@ -5,10 +5,7 @@
 #COBALT -A radix-io
 #COBALT -q ibleaf3-debug
 
-get_pmdk_lib_path() {
-        module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
-}
-
+export HOME=$SANDBOX
 export FI_FORK_UNSAFE=1
 
 . $SANDBOX/spack/share/spack/setup-env.sh
@@ -17,10 +14,6 @@ spack load -r bake
 
 module list
 
-# note, pmdk library not setting rpath correctly
-LIB_PATH_HACK=$(get_pmdk_lib_path)
-export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
-
 # echo "## MPI (one way, double the latency for round trip):"
 # mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency
 

perf-regression/cooley/mobject-regression.qsub

@@ -5,22 +5,13 @@
 #COBALT -A radix-io
 #COBALT -q ibleaf3-debug
 
-get_pmdk_lib_path() {
-        module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
-}
-
+export HOME=$SANDBOX
 export FI_FORK_UNSAFE=1
-
-export PATH=${SPACK_ROOT}/bin:${PATH}
-. ${SPACK_ROOT}/share/spack/setup-env.sh
+. ${SANDBOX}/spack/share/spack/setup-env.sh
 spack load -r ior@develop +mobject
 
 module list
 
-## note, pmdk library not setting rpath correctly
-#LIB_PATH_HACK=$(get_pmdk_lib_path)
-#export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
-
 # make half the nodes servers, half clients
 nnodes=$(wc -l <$COBALT_NODEFILE)
 head -n $((nnodes/2)) $COBALT_NODEFILE > server_nodes

perf-regression/cooley/pmdk-regression.qsub

@@ -5,10 +5,7 @@
 #COBALT -A radix-io
 #COBALT -q ibleaf3-debug
 
-get_pmdk_lib_path() {
-        module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
-}
-
+export HOME=$SANDBOX
 export FI_FORK_UNSAFE=1
 
 . $SANDBOX/spack/share/spack/setup-env.sh
@@ -17,10 +14,6 @@ spack load -r bake
 
 module list
 
-# note, pmdk library not setting rpath correctly
-LIB_PATH_HACK=$(get_pmdk_lib_path)
-export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
-
 echo "## PMDK (8x concurrency):"
 rm -f /dev/shm/foo.dat
 truncate -s 60G /dev/shm/foo.dat

perf-regression/cooley/run-regression.sh

 #!/bin/bash
 
 # This is a shell script to be run from a login node of the Cooley system at
-# the ALCF, that will download, compile, and execute the ssg performance 
+# the ALCF, that will download, compile, and execute Mochi performance 
 # regression tests, including any dependencies
 
-# SEE README.spack.md for environment setup information!  This script will not
-#     work properly without properly configured spack environment, in
-#     particular with a fix for the SPACK_SHELL environment variable detection
-
 # exit on any error
 set -e
 
+# grab helpful things (especially newer compiler) from softenv
+source /etc/profile.d/00softenv.sh
+soft add +gcc-8.2.0
+soft add +autotools-feb2016
+soft add +cmake-3.9.1
+soft add +mvapich2
+
+# location of this script
+ORIGIN=$PWD
+# scratch area for builds
 SANDBOX=$PWD/mochi-regression-sandbox-$$
+# install destination
 PREFIX=$PWD/mochi-regression-install-$$
+# job submission dir
 JOBDIR=$PWD/mochi-regression-job-$$
+# modify HOME env variable so that we don't perturb ~/.spack/ files for the 
+# users calling this script
+export HOME=$SANDBOX
 
-# scratch area to clone and build things
 mkdir $SANDBOX
-cp packages.yaml $SANDBOX/
-
-# scratch area for job submission
+mkdir $PREFIX
 mkdir $JOBDIR
-cp margo-regression.qsub $JOBDIR
-cp bake-regression.qsub $JOBDIR
-cp pmdk-regression.qsub $JOBDIR
-cp mobject-regression.qsub $JOBDIR
+cp $ORIGIN/margo-regression.qsub $JOBDIR
+cp $ORIGIN/bake-regression.qsub $JOBDIR
+cp $ORIGIN/pmdk-regression.qsub $JOBDIR
+cp $ORIGIN/mobject-regression.qsub $JOBDIR
 
+# set up build environment
 cd $SANDBOX
 git clone https://github.com/spack/spack.git
 git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
 git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git
-wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
-tar -xvzf osu-micro-benchmarks-5.3.2.tar.gz
-git clone https://github.com/pdlfs/mercury-runner.git
 
-# set up most of the libraries in spack
 echo "=== BUILD SPACK PACKAGES AND LOAD ==="
-export SPACK_ROOT=${SANDBOX}/spack
-cd $SANDBOX/spack
-. ${SPACK_ROOT}/share/spack/setup-env.sh
+. $SANDBOX/spack/share/spack/setup-env.sh
 spack compiler find
 spack compilers
-# put packages file in place in SPACK_ROOT to set our preferences for building
-# Mochi stack
-cp $SANDBOX/packages.yaml $SPACK_ROOT/etc/spack
-# set up repos file to point to sds-repo; we do this manually because 
-#    "spack repo add" will create files in ~/.spack, which is a bad idea in 
-#    CI environments
-echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml
-echo "- ${SANDBOX}/sds-repo" >> $SPACK_ROOT/etc/spack/repos.yaml
+
+# use our own packages.yaml for cooley-specific preferences
+cp $ORIGIN/packages.yaml $SPACK_ROOT/etc/spack
+# add external repo for mochi.  Note that this will not modify the 
+# user's ~/.spack/ files because we modified $HOME above
+spack repo add ${SANDBOX}/sds-repo
+# sanity check
+spack repo list
+# underlying tools needed by spack
 spack bootstrap
+# clean out any stray packages from previous runs, just in case
 spack uninstall -R -y argobots mercury rdma-core libfabric || true
+# ior acts as our "apex" package here, causing several other packages to build
 spack install ior@develop +mobject
 # deliberately repeat setup-env step after building modules to ensure
 #   that we pick up the right module paths
 . $SANDBOX/spack/share/spack/setup-env.sh
+# load ssg and bake because they are needed by things compiled outside of
+# spack later in this script
 spack load -r ssg
 spack load -r bake
 
-# less ancient gcc
-export CFLAGS="-O3"
-
-# OSU MPI benchmarks
-echo "=== BUILDING OSU MICRO BENCHMARKS ==="
-cd $SANDBOX/osu-micro-benchmarks-5.3.2
-mkdir build
-cd build
-../configure --prefix=$PREFIX CC=mpicc CXX=mpicxx
-make -j 3
-make install
-
 # sds-tests
 echo "=== BUILDING SDS TEST PROGRAMS ==="
 cd $SANDBOX/sds-tests
@@ -82,13 +78,13 @@ make -j 3
 make install
 
 # mercury-runner benchmark
-echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
-cd $SANDBOX/mercury-runner
-mkdir build
-cd build
-CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
-make -j 3
-make install
+# echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
+# cd $SANDBOX/mercury-runner
+# mkdir build
+# cd build
+# CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
+# make -j 3
+# make install
 
 # set up job to run
 echo "=== SUBMITTING AND WAITING FOR JOB ==="
@@ -96,16 +92,17 @@ cp $PREFIX/bin/margo-p2p-latency $JOBDIR
 cp $PREFIX/bin/margo-p2p-bw $JOBDIR
 cp $PREFIX/bin/bake-p2p-bw $JOBDIR
 cp $PREFIX/bin/pmdk-bw $JOBDIR
-cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR
-cp $PREFIX/bin/mercury-runner $JOBDIR
+# cp $PREFIX/bin/mercury-runner $JOBDIR
 cd $JOBDIR
-JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./margo-regression.qsub`
+
+# note: previously we also set --env LD_LIBRARY_PATH=$PREFIX/lib, hopefully no longer needed
+JOBID=`qsub --env SANDBOX=$SANDBOX ./margo-regression.qsub`
 cqwait $JOBID
-JOBID2=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./bake-regression.qsub`
+JOBID2=`qsub --env SANDBOX=$SANDBOX ./bake-regression.qsub`
 cqwait $JOBID2
-JOBID3=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
+JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
 cqwait $JOBID3
-JOBID4=`qsub --env SPACK_ROOT=${SPACK_ROOT} ./mobject-regression.qsub`
+JOBID4=`qsub --env SANDBOX=$SANDBOX ./mobject-regression.qsub`
 cqwait $JOBID4
 
 echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="