Commit 1c40dd29 authored by Philip Carns's avatar Philip Carns

revamp cooley regression scripts

- set $HOME to avoid perturbing ~/ config files for the user running this script
- remove deprecated --dirty and LD_PRELOAD hacks
- remove explicit set of SPACK_ROOT in favor of variable set by spack itself
- other misc streamlining improvements
parent 7ad45e79
To compile the full stack on cooley as of June 2018 using spack: To compile the full stack on cooley as of May 2019 using spack:
# set up .soft.cooley like this: 1. set up softenv by following the "soft add" commands in run-regression.sh, or add them to your .soft.cooley file
``` 2. get spack repos:
+gcc-7.1.0
+autotools-feb2016
+cmake-3.9.1
+mvapich2
@default
```
# get spack repos:
``` ```
git clone https://github.com/spack/spack.git git clone https://github.com/spack/spack.git
# as of 2018-06-12, need to pull in PR changes to allow SPACK_SHELL override git clone git@xgitlab.cels.anl.gov:sds/sds-repo
cd spack cd spack
git remote add jrood-nrel https://github.com/jrood-nrel/spack.git
git fetch --all
git merge jrood-nrel/fix_spack_shell_bootstrap
. share/spack/setup-env.sh . share/spack/setup-env.sh
spack bootstrap spack bootstrap
git clone git@xgitlab.cels.anl.gov:sds/sds-repo cd ../sds-repo
spack repo add . spack repo add .
``` ```
3. copy packages.yaml from this directory to your personal configuration location (usually ~/.spack/linux/)
This directory contains a packages.yaml that should be copied to one of the spack 4. compile and load the packages you want with `spack install` and `spack load`
configuration locations documented here: https://spack.readthedocs.io/en/stable/configuration.html . This could be your ~/.spack/linux/ directory, for example.
The automation scripts in this directory use $SPACK_ROOT/etc/spack to avoid
perturbing the account on which the regression tests are executed.
```
# compile everything and load module for ssg
# note that the --dirty option is needed because gcc 7.1 on cooley only works if it can inherit LD_LIBRARY_PATH from softenv
spack install --dirty ssg
spack load -r ssg
```
...@@ -5,10 +5,7 @@ ...@@ -5,10 +5,7 @@
#COBALT -A radix-io #COBALT -A radix-io
#COBALT -q ibleaf3-debug #COBALT -q ibleaf3-debug
get_pmdk_lib_path() { export HOME=$SANDBOX
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export FI_FORK_UNSAFE=1 export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh . $SANDBOX/spack/share/spack/setup-env.sh
...@@ -18,10 +15,6 @@ spack load -r mvapich2 ...@@ -18,10 +15,6 @@ spack load -r mvapich2
module list module list
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11" echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11"
echo "## Bake OFI/VERBS:" echo "## Bake OFI/VERBS:"
......
...@@ -5,10 +5,7 @@ ...@@ -5,10 +5,7 @@
#COBALT -A radix-io #COBALT -A radix-io
#COBALT -q ibleaf3-debug #COBALT -q ibleaf3-debug
get_pmdk_lib_path() { export HOME=$SANDBOX
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export FI_FORK_UNSAFE=1 export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh . $SANDBOX/spack/share/spack/setup-env.sh
...@@ -17,10 +14,6 @@ spack load -r bake ...@@ -17,10 +14,6 @@ spack load -r bake
module list module list
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
# echo "## MPI (one way, double the latency for round trip):" # echo "## MPI (one way, double the latency for round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency # mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency
......
...@@ -5,22 +5,13 @@ ...@@ -5,22 +5,13 @@
#COBALT -A radix-io #COBALT -A radix-io
#COBALT -q ibleaf3-debug #COBALT -q ibleaf3-debug
get_pmdk_lib_path() { export HOME=$SANDBOX
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export FI_FORK_UNSAFE=1 export FI_FORK_UNSAFE=1
. ${SANDBOX}/spack/share/spack/setup-env.sh
export PATH=${SPACK_ROOT}/bin:${PATH}
. ${SPACK_ROOT}/share/spack/setup-env.sh
spack load -r ior@develop +mobject spack load -r ior@develop +mobject
module list module list
## note, pmdk library not setting rpath correctly
#LIB_PATH_HACK=$(get_pmdk_lib_path)
#export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
# make half the nodes servers, half clients # make half the nodes servers, half clients
nnodes=$(wc -l <$COBALT_NODEFILE) nnodes=$(wc -l <$COBALT_NODEFILE)
head -n $((nnodes/2)) $COBALT_NODEFILE > server_nodes head -n $((nnodes/2)) $COBALT_NODEFILE > server_nodes
......
...@@ -5,10 +5,7 @@ ...@@ -5,10 +5,7 @@
#COBALT -A radix-io #COBALT -A radix-io
#COBALT -q ibleaf3-debug #COBALT -q ibleaf3-debug
get_pmdk_lib_path() { export HOME=$SANDBOX
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export FI_FORK_UNSAFE=1 export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh . $SANDBOX/spack/share/spack/setup-env.sh
...@@ -17,10 +14,6 @@ spack load -r bake ...@@ -17,10 +14,6 @@ spack load -r bake
module list module list
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo "## PMDK (8x concurrency):" echo "## PMDK (8x concurrency):"
rm -f /dev/shm/foo.dat rm -f /dev/shm/foo.dat
truncate -s 60G /dev/shm/foo.dat truncate -s 60G /dev/shm/foo.dat
......
#!/bin/bash #!/bin/bash
# This is a shell script to be run from a login node of the Cooley system at # This is a shell script to be run from a login node of the Cooley system at
# the ALCF, that will download, compile, and execute the ssg performance # the ALCF, that will download, compile, and execute Mochi performance
# regression tests, including any dependencies # regression tests, including any dependencies
# SEE README.spack.md for environment setup information! This script will not
# work properly without properly configured spack environment, in
# particular with a fix for the SPACK_SHELL environment variable detection
# exit on any error # exit on any error
set -e set -e
# grab helpful things (especially newer compiler) from softenv
source /etc/profile.d/00softenv.sh
soft add +gcc-8.2.0
soft add +autotools-feb2016
soft add +cmake-3.9.1
soft add +mvapich2
# location of this script
ORIGIN=$PWD
# scratch area for builds
SANDBOX=$PWD/mochi-regression-sandbox-$$ SANDBOX=$PWD/mochi-regression-sandbox-$$
# install destination
PREFIX=$PWD/mochi-regression-install-$$ PREFIX=$PWD/mochi-regression-install-$$
# job submission dir
JOBDIR=$PWD/mochi-regression-job-$$ JOBDIR=$PWD/mochi-regression-job-$$
# modify HOME env variable so that we don't perturb ~/.spack/ files for the
# users calling this script
export HOME=$SANDBOX
# scratch area to clone and build things
mkdir $SANDBOX mkdir $SANDBOX
cp packages.yaml $SANDBOX/ mkdir $PREFIX
# scratch area for job submission
mkdir $JOBDIR mkdir $JOBDIR
cp margo-regression.qsub $JOBDIR cp $ORIGIN/margo-regression.qsub $JOBDIR
cp bake-regression.qsub $JOBDIR cp $ORIGIN/bake-regression.qsub $JOBDIR
cp pmdk-regression.qsub $JOBDIR cp $ORIGIN/pmdk-regression.qsub $JOBDIR
cp mobject-regression.qsub $JOBDIR cp $ORIGIN/mobject-regression.qsub $JOBDIR
# set up build environment
cd $SANDBOX cd $SANDBOX
git clone https://github.com/spack/spack.git git clone https://github.com/spack/spack.git
git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git
wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
tar -xvzf osu-micro-benchmarks-5.3.2.tar.gz
git clone https://github.com/pdlfs/mercury-runner.git
# set up most of the libraries in spack
echo "=== BUILD SPACK PACKAGES AND LOAD ===" echo "=== BUILD SPACK PACKAGES AND LOAD ==="
export SPACK_ROOT=${SANDBOX}/spack . $SANDBOX/spack/share/spack/setup-env.sh
cd $SANDBOX/spack
. ${SPACK_ROOT}/share/spack/setup-env.sh
spack compiler find spack compiler find
spack compilers spack compilers
# put packages file in place in SPACK_ROOT to set our preferences for building
# Mochi stack # use our own packages.yaml for cooley-specific preferences
cp $SANDBOX/packages.yaml $SPACK_ROOT/etc/spack cp $ORIGIN/packages.yaml $SPACK_ROOT/etc/spack
# set up repos file to point to sds-repo; we do this manually because # add external repo for mochi. Note that this will not modify the
# "spack repo add" will create files in ~/.spack, which is a bad idea in # user's ~/.spack/ files because we modified $HOME above
# CI environments spack repo add ${SANDBOX}/sds-repo
echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml # sanity check
echo "- ${SANDBOX}/sds-repo" >> $SPACK_ROOT/etc/spack/repos.yaml spack repo list
# underlying tools needed by spack
spack bootstrap spack bootstrap
# clean out any stray packages from previous runs, just in case
spack uninstall -R -y argobots mercury rdma-core libfabric || true spack uninstall -R -y argobots mercury rdma-core libfabric || true
# ior acts as our "apex" package here, causing several other packages to build
spack install ior@develop +mobject spack install ior@develop +mobject
# deliberately repeat setup-env step after building modules to ensure # deliberately repeat setup-env step after building modules to ensure
# that we pick up the right module paths # that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh . $SANDBOX/spack/share/spack/setup-env.sh
# load ssg and bake because they are needed by things compiled outside of
# spack later in this script
spack load -r ssg spack load -r ssg
spack load -r bake spack load -r bake
# less ancient gcc
export CFLAGS="-O3"
# OSU MPI benchmarks
echo "=== BUILDING OSU MICRO BENCHMARKS ==="
cd $SANDBOX/osu-micro-benchmarks-5.3.2
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc CXX=mpicxx
make -j 3
make install
# sds-tests # sds-tests
echo "=== BUILDING SDS TEST PROGRAMS ===" echo "=== BUILDING SDS TEST PROGRAMS ==="
cd $SANDBOX/sds-tests cd $SANDBOX/sds-tests
...@@ -82,13 +78,13 @@ make -j 3 ...@@ -82,13 +78,13 @@ make -j 3
make install make install
# mercury-runner benchmark # mercury-runner benchmark
echo "=== BUILDING MERCURY-RUNNER BENCHMARK ===" # echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
cd $SANDBOX/mercury-runner # cd $SANDBOX/mercury-runner
mkdir build # mkdir build
cd build # cd build
CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON .. # CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
make -j 3 # make -j 3
make install # make install
# set up job to run # set up job to run
echo "=== SUBMITTING AND WAITING FOR JOB ===" echo "=== SUBMITTING AND WAITING FOR JOB ==="
...@@ -96,16 +92,17 @@ cp $PREFIX/bin/margo-p2p-latency $JOBDIR ...@@ -96,16 +92,17 @@ cp $PREFIX/bin/margo-p2p-latency $JOBDIR
cp $PREFIX/bin/margo-p2p-bw $JOBDIR cp $PREFIX/bin/margo-p2p-bw $JOBDIR
cp $PREFIX/bin/bake-p2p-bw $JOBDIR cp $PREFIX/bin/bake-p2p-bw $JOBDIR
cp $PREFIX/bin/pmdk-bw $JOBDIR cp $PREFIX/bin/pmdk-bw $JOBDIR
cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR # cp $PREFIX/bin/mercury-runner $JOBDIR
cp $PREFIX/bin/mercury-runner $JOBDIR
cd $JOBDIR cd $JOBDIR
JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./margo-regression.qsub`
# note: previously we also set --env LD_LIBRARY_PATH=$PREFIX/lib, hopefully no longer needed
JOBID=`qsub --env SANDBOX=$SANDBOX ./margo-regression.qsub`
cqwait $JOBID cqwait $JOBID
JOBID2=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./bake-regression.qsub` JOBID2=`qsub --env SANDBOX=$SANDBOX ./bake-regression.qsub`
cqwait $JOBID2 cqwait $JOBID2
JOBID3=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./pmdk-regression.qsub` JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
cqwait $JOBID3 cqwait $JOBID3
JOBID4=`qsub --env SPACK_ROOT=${SPACK_ROOT} ./mobject-regression.qsub` JOBID4=`qsub --env SANDBOX=$SANDBOX ./mobject-regression.qsub`
cqwait $JOBID4 cqwait $JOBID4
echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ===" echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment