Commit 0b4b28f3 authored by Philip Carns's avatar Philip Carns

update bebop regression tests

- some examples are crashing at runtime, not sure why yet
parent 1c40dd29
......@@ -18,11 +18,8 @@ spack load -r bake
module list
# force Intel MPI to use Infiniband instead of Omnipath/PSM2. If we let
# Intel MPI use it's preferred PSM2 path then Mercury can't open interface
# because of apparent MR_SCALABLE/MR_BASIC conflict
# we should also try MR_SCALABLE in Mercury NA as a possible workaround
export I_MPI_FABRICS=shm:ofa
# make sure that MPI and libfabric can share PSM2
export PSM2_MULTI_EP=1
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on bebop nodes: cores 18-35"
......
......@@ -14,14 +14,12 @@ module load numactl boost cmake
. $SANDBOX/spack/share/spack/setup-env.sh
spack load -r ssg
spack load -r bake
module list
# force Intel MPI to use Infiniband instead of Omnipath/PSM2. If we let
# Intel MPI use it's preferred PSM2 path then Mercury can't open interface
# because of apparent MR_SCALABLE/MR_BASIC conflict
# we should also try MR_SCALABLE in Mercury NA as a possible workaround
export I_MPI_FABRICS=shm:ofa
# make sure that MPI and libfabric can share PSM2
export PSM2_MULTI_EP=1
echo "## Margo OFI/PSM2 (round trip, bdw):"
mpirun ./margo-p2p-latency -i 100000 -n psm2://enp6s0f0:5000
......@@ -37,10 +35,10 @@ echo "## Margo OFI/PSM2 (bw, 1MB unaligned, bdw):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 1 -D 20
echo "## Margo OFI/PSM2 (bw, 1MB unaligned, 8x concurrency, bdw):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 8 -D 20
echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, bdw):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -m /dev/shm/foo
echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, 8x concurrency, bdw):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -m /dev/shm/foo
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, bdw):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -m /dev/shm/foo
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, 8x concurrency, bdw):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -m /dev/shm/foo
echo "## Margo OFI/PSM2 (round trip, bdw, Hg busy spin):"
mpirun ./margo-p2p-latency -i 100000 -n psm2://enp6s0f0:5000 -t 0,0
......@@ -56,8 +54,8 @@ echo "## Margo OFI/PSM2 (bw, 1MB unaligned, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 1MB unaligned, 8x concurrency, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 1000000 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0
echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0 -m /dev/shm/foo
echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, 8x concurrency, bdw, Hg busy spin):"
mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0 -m /dev/shm/foo
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, bdw, Hg busy spin):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 1 -D 20 -t 0,0 -m /dev/shm/foo
# echo "## Margo OFI/PSM2 (bw, 8MiB, mmap shmfs on svr, 8x concurrency, bdw, Hg busy spin):"
# mpirun ./margo-p2p-bw -x 8388608 -n psm2://enp6s0f0:5000 -c 8 -D 20 -t 0,0 -m /dev/shm/foo
......@@ -15,13 +15,8 @@ spack load -r ior
module list
# force Intel MPI to use Infiniband instead of Omnipath/PSM2. If we let
# Intel MPI use it's preferred PSM2 path then Mercury can't open interface
# because of apparent MR_SCALABLE/MR_BASIC conflict
# we should also try MR_SCALABLE in Mercury NA as a possible workaround
export I_MPI_FABRICS=shm:ofa
# make sure that MPI and libfabric can share PSM2
export PSM2_MULTI_EP=1
export MOBJECT_CLUSTER_FILE=mobject-cluster.gid
echo " ==== running server"
......
......@@ -18,11 +18,8 @@ spack load -r bake
module list
# force Intel MPI to use Infiniband instead of Omnipath/PSM2. If we let
# Intel MPI use it's preferred PSM2 path then Mercury can't open interface
# because of apparent MR_SCALABLE/MR_BASIC conflict
# we should also try MR_SCALABLE in Mercury NA as a possible workaround
export I_MPI_FABRICS=shm:ofa
# make sure that MPI and libfabric can share PSM2
export PSM2_MULTI_EP=1
echo "## PMDK (8x concurrency):"
rm -f /dev/shm/foo.dat
......
......@@ -10,105 +10,90 @@
# exit on any error
set -e
get_psm2_lib_path() {
module show `spack module tcl find opa-psm2` |&grep LD_LIBRARY_PATH | cut -d \" -f 4
}
# load newer gcc up front
module load gcc_new/7.3.0
# location of this script
ORIGIN=$PWD
# scratch area for builds
SANDBOX=$PWD/mochi-regression-sandbox-$$
# install destination
PREFIX=$PWD/mochi-regression-install-$$
# job submission dir
JOBDIR=$PWD/mochi-regression-job-$$
# scratch area to clone and build things
mkdir -p $SANDBOX
cp packages.yaml $SANDBOX/
# scratch area for job submission
mkdir -p $JOBDIR
cp margo-regression.sbatch $JOBDIR
cp bake-regression.sbatch $JOBDIR
cp pmdk-regression.sbatch $JOBDIR
cp mobject-regression.sbatch $JOBDIR
# modify HOME env variable so that we don't perturb ~/.spack/ files for the
# users calling this script
export HOME=$SANDBOX
mkdir $SANDBOX
mkdir $PREFIX
mkdir $JOBDIR
cp $ORIGIN/margo-regression.sbatch $JOBDIR
cp $ORIGIN/bake-regression.sbatch $JOBDIR
cp $ORIGIN/pmdk-regression.sbatch $JOBDIR
cp $ORIGIN/mobject-regression.sbatch $JOBDIR
# set up build environment
cd $SANDBOX
git clone https://github.com/spack/spack.git
# back out prolematic opa-psm2 change
(cd spack && git revert --no-edit d4fbaa054f8d718eaffecb53b1d37ad93ceeca05)
# as of 2019-05-17, clone fork with correction to PSM2 package
git clone -b carns/dev-opa-psm2-path https://github.com/carns/spack.git
git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
git clone https://xgitlab.cels.anl.gov/sds/sds-tests.git
wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
tar -xvzf osu-micro-benchmarks-5.3.2.tar.gz
git clone https://github.com/pdlfs/mercury-runner.git
export SPACK_ROOT=${SANDBOX}/spack
# set up most of the libraries in spack
echo "=== BUILD SPACK PACKAGES AND LOAD ==="
cd $SANDBOX/spack
. $SANDBOX/spack/share/spack/setup-env.sh
spack compiler find
spack compilers
# put packages file in place in SPACK_ROOT to set our preferences for building
# Mochi stack
mkdir -p $SPACK_ROOT/etc/spack
cp $SANDBOX/packages.yaml $SPACK_ROOT/etc/spack
# set up repos file to point to sds-repo; we do this manually because
# "spack repo add" will create files in ~/.spack, which is a bad idea in
# CI environments
echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml
echo "- ${SANDBOX}/sds-repo" >> $SPACK_ROOT/etc/spack/repos.yaml
# use our own packages.yaml for bebop-specific preferences
cp $ORIGIN/packages.yaml $SPACK_ROOT/etc/spack
# add external repo for mochi. Note that this will not modify the
# user's ~/.spack/ files because we modified $HOME above
spack repo add ${SANDBOX}/sds-repo
# sanity check
spack repo list
# clean out any stray packages from previous runs, just in case
spack uninstall -R -y argobots mercury opa-psm2 bake || true
# nightly tests should test nightly software!
spack install ior@develop+mobject ^margo@develop ^mercury@develop ^mobject@develop ^bake@develop ^remi@develop ^thallium@develop ^sdskeyval@develop ^ssg@develop
# spack install ior@develop+mobject ^margo@develop ^mercury@develop ^mobject@develop ^bake@develop ^remi@develop ^thallium@develop ^sdskeyval@develop ^ssg@develop
# ior acts as our "apex" package here, causing several other packages to build
spack install ior@develop +mobject
# deliberately repeat setup-env step after building modules to ensure
# that we pick up the right module paths
. $SANDBOX/spack/share/spack/setup-env.sh
# load ssg and bake because they are needed by things compiled outside of
# spack later in this script
spack load -r ssg
spack load -r bake
spack load -r mobject
export CFLAGS="-O3"
# OSU MPI benchmarks
echo "=== BUILDING OSU MICRO BENCHMARKS ==="
cd $SANDBOX/osu-micro-benchmarks-5.3.2
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc CXX=mpicxx
make -j 3
make install
# sds-tests
echo "=== BUILDING SDS TEST PROGRAMS ==="
# TODO: why is this needed? For some reason when we link software
# outside of spack we are not picking up the path from the psm2 package
LIB_PATH_HACK=$(get_psm2_lib_path)
cd $SANDBOX/sds-tests
libtoolize
./prepare.sh
mkdir build
cd build
../configure --prefix=$PREFIX CC=mpicc
echo ../configure --prefix=$PREFIX CC=mpicc LDFLAGS="-L$LIB_PATH_HACK"
../configure --prefix=$PREFIX CC=mpicc LDFLAGS="-L$LIB_PATH_HACK"
make -j 3
make install
# mercury-runner benchmark
# echo "=== BUILDING MERCURY-RUNNER BENCHMARK ==="
# cd $SANDBOX/mercury-runner
# mkdir build
# cd build
# CC=mpicc CXX=mpicxx CXXFLAGS='-D__STDC_FORMAT_MACROS' cmake -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=$PREFIX -DMPI=ON ..
# make -j 3
# make install
# set up job to run
echo "=== SUBMITTING AND WAITING FOR JOB ==="
cp $PREFIX/bin/margo-p2p-latency $JOBDIR
cp $PREFIX/bin/margo-p2p-bw $JOBDIR
cp $PREFIX/bin/bake-p2p-bw $JOBDIR
cp $PREFIX/bin/pmdk-bw $JOBDIR
cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR
# cp $PREFIX/bin/mercury-runner $JOBDIR
cd $JOBDIR
export SANDBOX
sbatch --wait --export=ALL ./margo-regression.sbatch
sbatch --wait --export=ALL ./bake-regression.sbatch
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment