GitLab maintenance scheduled for Today, 2019-12-05, from 17:00 to 18:00 CT - Services will be unavailable during this time.

Commit 1c40dd29 authored by Philip Carns's avatar Philip Carns

revamp cooley regression scripts

- set $HOME to avoid perturbing ~/ config files for the user running this script
- remove deprecated --dirty and LD_PRELOAD hacks
- remove explicit set of SPACK_ROOT in favor of variable set by spack itself
- other misc streamlining improvements
parent 7ad45e79
To compile the full stack on cooley as of June 2018 using spack:
To compile the full stack on cooley as of May 2019 using spack:
# set up .soft.cooley like this:
```
+gcc-7.1.0
+autotools-feb2016
+cmake-3.9.1
+mvapich2
@default
```
# get spack repos:
1. set up softenv by following the "soft add" commands in run-regression.sh, or add them to your .soft.cooley file
2. get spack repos:
```
git clone https://github.com/spack/spack.git
# as of 2018-06-12, need to pull in PR changes to allow SPACK_SHELL override
git clone git@xgitlab.cels.anl.gov:sds/sds-repo
cd spack
git remote add jrood-nrel https://github.com/jrood-nrel/spack.git
git fetch --all
git merge jrood-nrel/fix_spack_shell_bootstrap
. share/spack/setup-env.sh
spack bootstrap
git clone git@xgitlab.cels.anl.gov:sds/sds-repo
cd ../sds-repo
spack repo add .
```
This directory contains a packages.yaml that should be copied to one of the spack
configuration locations documented here: https://spack.readthedocs.io/en/stable/configuration.html . This could be your ~/.spack/linux/ directory, for example.
The automation scripts in this directory use $SPACK_ROOT/etc/spack to avoid
perturbing the account on which the regression tests are executed.
```
# compile everything and load module for ssg
# note that the --dirty option is needed because gcc 7.1 on cooley only works if it can inherit LD_LIBRARY_PATH from softenv
spack install --dirty ssg
spack load -r ssg
```
3. copy packages.yaml from this directory to your personal configuration location (usually ~/.spack/linux/)
4. compile and load the packages you want with `spack install` and `spack load`
......@@ -5,10 +5,7 @@
#COBALT -A radix-io
#COBALT -q ibleaf3-debug
get_pmdk_lib_path() {
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh
......@@ -18,10 +15,6 @@ spack load -r mvapich2
module list
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11"
echo "## Bake OFI/VERBS:"
......
......@@ -5,10 +5,7 @@
#COBALT -A radix-io
#COBALT -q ibleaf3-debug
get_pmdk_lib_path() {
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh
......@@ -17,10 +14,6 @@ spack load -r bake
module list
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
# echo "## MPI (one way, double the latency for round trip):"
# mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency
......
......@@ -5,22 +5,13 @@
#COBALT -A radix-io
#COBALT -q ibleaf3-debug
get_pmdk_lib_path() {
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
export PATH=${SPACK_ROOT}/bin:${PATH}
. ${SPACK_ROOT}/share/spack/setup-env.sh
. ${SANDBOX}/spack/share/spack/setup-env.sh
spack load -r ior@develop +mobject
module list
## note, pmdk library not setting rpath correctly
#LIB_PATH_HACK=$(get_pmdk_lib_path)
#export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
# make half the nodes servers, half clients
nnodes=$(wc -l <$COBALT_NODEFILE)
head -n $((nnodes/2)) $COBALT_NODEFILE > server_nodes
......
......@@ -5,10 +5,7 @@
#COBALT -A radix-io
#COBALT -q ibleaf3-debug
get_pmdk_lib_path() {
module show `spack module tcl find pmdk` |&grep LIBRARY_PATH | awk '{print $3}'
}
export HOME=$SANDBOX
export FI_FORK_UNSAFE=1
. $SANDBOX/spack/share/spack/setup-env.sh
......@@ -17,10 +14,6 @@ spack load -r bake
module list
# note, pmdk library not setting rpath correctly
LIB_PATH_HACK=$(get_pmdk_lib_path)
export LD_LIBRARY_PATH="$LIB_PATH_HACK:$LD_LIBRARY_PATH"
echo "## PMDK (8x concurrency):"
rm -f /dev/shm/foo.dat
truncate -s 60G /dev/shm/foo.dat
......
#!/bin/bash
# This is a shell script to be run from a login node of the Cooley system at
# the ALCF, that will download, compile, and execute the ssg performance
# the ALCF, that will download, compile, and execute Mochi performance
# regression tests, including any dependencies
# SEE README.spack.md for environment setup information! This script will not
# work properly without properly configured spack environment, in
# particular with a fix for the SPACK_SHELL environment variable detection
# exit on any error
set -e
# grab helpful things (especially newer compiler) from softenv
source /etc/profile.d/00softenv.sh
soft add +gcc-8.2.0
soft add +autotools-feb2016
soft add +cmake-3.9.1
soft add +mvapich2
# location of this script
ORIGIN=$PWD
# scratch area for builds
SANDBOX=$PWD/mochi-regression-sandbox-$$
# install destination
PREFIX=$PWD/mochi-regression-install-$$
# job submission dir
JOBDIR=$PWD/mochi-regression-job-$$
# modify HOME env variable so that we don't perturb ~/.spack/ files for the
# users calling this script
export HOME=$SANDBOX
# scratch area to clone and build things
mkdir $SANDBOX
cp packages.yaml $SANDBOX/
# scratch area for job submission
mkdir $PREFIX
mkdir $JOBDIR
cp margo-regression.qsub $JOBDIR
cp bake-regression.qsub $JOBDIR
cp pmdk-regression.qsub $JOBDIR
cp mobject-regression.qsub $JOBDIR
cp $ORIGIN/margo-regression.qsub $JOBDIR
cp $ORIGIN/bake-regression.qsub $JOBDIR
cp $ORIGIN/pmdk-regression.qsub $JOBDIR
cp $ORIGIN/mobject-regression.qsub $JOBDIR
# set up build environment
cd $SANDBOX
git clone https://github.com/spack/spack.git
git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git
wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
tar -xvzf osu-micro-benchmarks-5.3.2.tar.gz
git clone https://github.com/pdlfs/mercury-runner.git
# set up most of the libraries in spack
echo "=== BUILD SPACK PACKAGES AND LOAD ==="
export SPACK_ROOT=${SANDBOX}/spack
cd $SANDBOX/spack
. ${SPACK_ROOT}/share/spack/setup-env.sh
. $SANDBOX/spack/share/spack/setup-env.sh
spack compiler find
spack compilers
# put packages file in place in SPACK_ROOT to set our preferences for building
# Mochi stack
cp $SANDBOX/packages.yaml $SPACK_ROOT/etc/spack
# set up repos file to point to sds-repo; we do this manually because
# "spack repo add" will create files in ~/.spack, which is a bad idea in
# CI environments
echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml
echo "- ${SANDBOX}/sds-repo" >> $SPACK_ROOT/etc/spack/repos.yaml
# use our own packages.yaml for cooley-specific preferences
cp $ORIGIN/packages.yaml $SPACK_ROOT/etc/spack
# add external repo for mochi. Note that this will not modify the
# user's ~/.spack/ files because we modified $HOME above
spack repo add ${SANDBOX}/sds-repo
# sanity check
spack repo list
# underlying tools needed by spack
spack bootstrap
# clean out any stray packages from previous runs, just in case
spack uninstall -R -y argobots mercury rdma-core libfabric || true
# ior acts as our "apex" package here, causing several other packages to build
spack install ior@develop +mobject
# deliberately repeat setup-env step after building modules to ensure
# that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh
# load ssg and bake because they are needed by things compiled outside of
# spack later in this script
spack load -r ssg
spack load -r bake
# less ancient gcc
export CFLAGS="-O3"
# OSU MPI benchmarks
echo "=== BUILDING OSU MICRO BENCHMARKS ==="
cd $SANDBOX/osu-micro-benchmarks-5.3.2
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc CXX=mpicxx
make -j 3
make install
# sds-tests
echo "=== BUILDING SDS TEST PROGRAMS ==="
cd $SANDBOX/sds-tests
......@@ -82,13 +78,13 @@ make -j 3
make install
# mercury-runner benchmark
echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
cd $SANDBOX/mercury-runner
mkdir build
cd build
CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
make -j 3
make install
# echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
# cd $SANDBOX/mercury-runner
# mkdir build
# cd build
# CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
# make -j 3
# make install
# set up job to run
echo "=== SUBMITTING AND WAITING FOR JOB ==="
......@@ -96,16 +92,17 @@ cp $PREFIX/bin/margo-p2p-latency $JOBDIR
cp $PREFIX/bin/margo-p2p-bw $JOBDIR
cp $PREFIX/bin/bake-p2p-bw $JOBDIR
cp $PREFIX/bin/pmdk-bw $JOBDIR
cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR
cp $PREFIX/bin/mercury-runner $JOBDIR
# cp $PREFIX/bin/mercury-runner $JOBDIR
cd $JOBDIR
JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./margo-regression.qsub`
# note: previously we also set --env LD_LIBRARY_PATH=$PREFIX/lib, hopefully no longer needed
JOBID=`qsub --env SANDBOX=$SANDBOX ./margo-regression.qsub`
cqwait $JOBID
JOBID2=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./bake-regression.qsub`
JOBID2=`qsub --env SANDBOX=$SANDBOX ./bake-regression.qsub`
cqwait $JOBID2
JOBID3=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
cqwait $JOBID3
JOBID4=`qsub --env SPACK_ROOT=${SPACK_ROOT} ./mobject-regression.qsub`
JOBID4=`qsub --env SANDBOX=$SANDBOX ./mobject-regression.qsub`
cqwait $JOBID4
echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment