run-regression-ofi-rxm.sh 3.68 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
#!/bin/bash

# This is a shell script to be run from a login node of the Cooley system at
# the ALCF, that will download, compile, and execute the ssg performance 
# regression tests, including any dependencies

# SEE README.spack.md for environment setup information!  This script will not
#     work properly without properly configured spack environment, in
#     particular with a fix for the SPACK_SHELL environment variable detection

# exit on any error
set -e

Philip Carns's avatar
Philip Carns committed
14 15 16
SANDBOX=$PWD/mochi-regression-sandbox-$$
PREFIX=$PWD/mochi-regression-install-$$
JOBDIR=$PWD/mochi-regression-job-$$
17 18 19

# scratch area to clone and build things
mkdir $SANDBOX
Philip Carns's avatar
Philip Carns committed
20
cp packages.yaml $SANDBOX/
21 22 23 24

# scratch area for job submission
mkdir $JOBDIR
cp margo-regression-ofi-rxm.qsub $JOBDIR
25
cp bake-regression.qsub $JOBDIR
26 27

cd $SANDBOX
28
git clone https://github.com/carns/spack.git
29
git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
30
git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git
31 32 33 34 35 36 37
wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
tar -xvzf osu-micro-benchmarks-5.3.2.tar.gz
git clone https://github.com/pdlfs/mercury-runner.git

# set up most of the libraries in spack
echo "=== BUILD SPACK PACKAGES AND LOAD ==="
cd $SANDBOX/spack
38
git checkout carns/dev-shell-detection
39
. $SANDBOX/spack/share/spack/setup-env.sh
Philip Carns's avatar
Philip Carns committed
40 41 42 43 44 45 46 47
# put packages file in place in SPACK_ROOT to set our preferences for building
# Mochi stack
cp $SANDBOX/packages.yaml $SPACK_ROOT/etc/spack
# set up repos file to point to sds-repo; we do this manually because 
#    "spack repo add" will create files in ~/.spack, which is a bad idea in 
#    CI environments
echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml
echo "- ${SANDBOX}/sds-repo" >> $SPACK_ROOT/etc/spack/repos.yaml
48 49
spack bootstrap
spack uninstall -R -y argobots mercury rdma-core libfabric || true
50 51
# 2018-11-5 using @develop for mercury and libfabric to test wait_fd
spack install --dirty ssg ^mercury@develop ^libfabric@develop
52
spack install --dirty bake ^mercury@develop ^libfabric@develop
53 54 55 56
# deliberately repeat setup-env step after building modules to ensure
#   that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg
57
spack load -r bake
58 59 60 61 62 63 64 65 66 67 68 69 70

# less ancient gcc
export CFLAGS="-O3"

# OSU MPI benchmarks
echo "=== BUILDING OSU MICRO BENCHMARKS ==="
cd $SANDBOX/osu-micro-benchmarks-5.3.2
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc CXX=mpicxx
make -j 3
make install

71 72 73
# sds-tests
echo "=== BUILDING SDS TEST PROGRAMS ==="
cd $SANDBOX/sds-tests
74 75 76 77 78 79
libtoolize
./prepare.sh
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc
make -j 3
80
make install
81 82 83 84 85 86 87 88 89 90 91 92

# mercury-runner benchmark
echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
cd $SANDBOX/mercury-runner
mkdir build
cd build
CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
make -j 3
make install

# set up job to run
echo "=== SUBMITTING AND WAITING FOR JOB ==="
93 94
cp $PREFIX/bin/margo-p2p-latency $JOBDIR
cp $PREFIX/bin/margo-p2p-bw $JOBDIR
95
cp $PREFIX/bin/bake-p2p-bw $JOBDIR
96 97 98 99 100
cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR
cp $PREFIX/bin/mercury-runner $JOBDIR
cd $JOBDIR
JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./margo-regression-ofi-rxm.qsub`
cqwait $JOBID
101 102
JOBID2=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib --env SANDBOX=$SANDBOX ./bake-regression.qsub`
cqwait $JOBID2
103 104 105

echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
# gather output, strip out funny characters, mail
106
cat $JOBID.* $JOBID2.* > combined.$JOBID.txt
107
#dos2unix combined.$JOBID.txt
108
mailx -s "margo-regression (cooley)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
109 110 111 112

cd /tmp
rm -rf $SANDBOX
rm -rf $PREFIX