run-regression.sh 4.45 KB
Newer Older
1 2 3
#!/bin/bash

# This is a shell script to be run from a login node of the Cooley system at
4
# the ALCF, that will download, compile, and execute Mochi performance 
5 6 7 8 9
# regression tests, including any dependencies

# exit on any error
set -e

10 11 12 13 14 15 16 17 18 19
# grab helpful things (especially newer compiler) from softenv
source /etc/profile.d/00softenv.sh
soft add +gcc-8.2.0
soft add +autotools-feb2016
soft add +cmake-3.9.1
soft add +mvapich2

# location of this script
ORIGIN=$PWD
# scratch area for builds
Philip Carns's avatar
Philip Carns committed
20
SANDBOX=$PWD/mochi-regression-sandbox-$$
21
# install destination
Philip Carns's avatar
Philip Carns committed
22
PREFIX=$PWD/mochi-regression-install-$$
23
# job submission dir
Philip Carns's avatar
Philip Carns committed
24
JOBDIR=$PWD/mochi-regression-job-$$
25 26 27
# modify HOME env variable so that we don't perturb ~/.spack/ files for the 
# users calling this script
export HOME=$SANDBOX
28 29

mkdir $SANDBOX
30
mkdir $PREFIX
31
mkdir $JOBDIR
32 33 34 35
cp $ORIGIN/margo-regression.qsub $JOBDIR
cp $ORIGIN/bake-regression.qsub $JOBDIR
cp $ORIGIN/pmdk-regression.qsub $JOBDIR
cp $ORIGIN/mobject-regression.qsub $JOBDIR
36
cp $ORIGIN/bake-kove.qsub $JOBDIR
37

38
# set up build environment
39
cd $SANDBOX
40
git clone https://github.com/spack/spack.git
41
git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
42
git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git
43 44

echo "=== BUILD SPACK PACKAGES AND LOAD ==="
45
. $SANDBOX/spack/share/spack/setup-env.sh
46 47
spack compiler find
spack compilers
48 49 50 51 52 53 54 55 56

# use our own packages.yaml for cooley-specific preferences
cp $ORIGIN/packages.yaml $SPACK_ROOT/etc/spack
# add external repo for mochi.  Note that this will not modify the 
# user's ~/.spack/ files because we modified $HOME above
spack repo add ${SANDBOX}/sds-repo
# sanity check
spack repo list
# underlying tools needed by spack
57
spack bootstrap
58
# clean out any stray packages from previous runs, just in case
59
spack uninstall -R -y argobots mercury rdma-core libfabric || true
60
# ior acts as our "apex" package here, causing several other packages to build
61
spack install ior@develop +mobject
Philip Carns's avatar
Philip Carns committed
62 63
spack install mochi-ssg
spack install mochi-bake
64
# check what stable version of bake we got
Philip Carns's avatar
Philip Carns committed
65
BAKE_STABLE_VER=`spack find mochi-bake |grep mochi-bake |grep -v file-backend`
66
# load an additional version of bake that uses a file backend
67
# temporary: thallium@develop contains fixes for recent argobots opaque type changes
Philip Carns's avatar
Philip Carns committed
68
spack install mochi-bake@dev-file-backend ^mochi-thallium@develop
69 70 71
# deliberately repeat setup-env step after building modules to ensure
#   that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh
72 73
# load ssg and bake because they are needed by things compiled outside of
# spack later in this script
Philip Carns's avatar
Philip Carns committed
74
spack load -r mochi-ssg
75
spack load -r $BAKE_STABLE_VER
76

77 78 79
# sds-tests
echo "=== BUILDING SDS TEST PROGRAMS ==="
cd $SANDBOX/sds-tests
80 81 82 83 84 85
libtoolize
./prepare.sh
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc
make -j 3
86
make install
87

88 89 90
# switch bake versions and build another copy with file backend
echo "=== BUILDING SDS TEST PROGRAMS WITH FILE BACKEND ==="
spack unload $BAKE_STABLE_VER
Philip Carns's avatar
Philip Carns committed
91
spack load -r mochi-bake@dev-file-backend
92 93 94 95 96 97 98
cd $SANDBOX/sds-tests
mkdir build-file
cd build-file
../configure --prefix=${PREFIX}-file CC=mpicc
make -j 3
make install

99
# mercury-runner benchmark
100 101 102 103 104 105 106
# echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
# cd $SANDBOX/mercury-runner
# mkdir build
# cd build
# CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
# make -j 3
# make install
107 108 109

# set up job to run
echo "=== SUBMITTING AND WAITING FOR JOB ==="
110 111
cp $PREFIX/bin/margo-p2p-latency $JOBDIR
cp $PREFIX/bin/margo-p2p-bw $JOBDIR
112
cp $PREFIX/bin/bake-p2p-bw $JOBDIR
113
cp ${PREFIX}-file/bin/bake-p2p-bw $JOBDIR/bake-p2p-bw-file
114
cp $PREFIX/bin/pmdk-bw $JOBDIR
115
# cp $PREFIX/bin/mercury-runner $JOBDIR
116
cd $JOBDIR
117 118 119

# note: previously we also set --env LD_LIBRARY_PATH=$PREFIX/lib, hopefully no longer needed
JOBID=`qsub --env SANDBOX=$SANDBOX ./margo-regression.qsub`
120
cqwait $JOBID
121
JOBID2=`qsub --env SANDBOX=$SANDBOX ./bake-regression.qsub`
122
cqwait $JOBID2
123
JOBID3=`qsub --env SANDBOX=$SANDBOX ./pmdk-regression.qsub`
124
cqwait $JOBID3
125 126
JOBID4=`qsub --env SANDBOX=$SANDBOX ./mobject-regression.qsub`
cqwait $JOBID4
127 128
JOBID5=`qsub --env SANDBOX=$SANDBOX ./bake-kove.qsub`
cqwait $JOBID5
129 130 131

echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
# gather output, strip out funny characters, mail
132
cat $JOBID.* $JOBID2.* $JOBID3.* $JOBID4.* $JOBID5.* > combined.$JOBID.txt
133
dos2unix combined.$JOBID.txt
134
mailx -s "mochi-regression (cooley)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
135 136

cd /tmp
Philip Carns's avatar
Philip Carns committed
137 138
rm -rf $SANDBOX
rm -rf $PREFIX