Commit fa8b85ae authored by Philip Carns's avatar Philip Carns

get bake benchmarks running on theta

- requires 'udreg' memory registration method
- works with libfabric 1.7
- use patches for now until changes upstreamed
- packages.yaml tweaked to reduce build time
parent 95b320cb
......@@ -8,6 +8,9 @@
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt
# necessary when using the udreg option in Mercury
export MPICH_GNI_NDREG_ENTRIES=1024
module swap PrgEnv-intel PrgEnv-gnu
module load cce
......
......@@ -8,6 +8,9 @@
# XXX xalt module currently eating '-M' flag for mercury-runner...disabling for now
# module unload xalt
# necessary when using the udreg option in Mercury
export MPICH_GNI_NDREG_ENTRIES=1024
module swap PrgEnv-intel PrgEnv-gnu
module load cce
......
......@@ -51,6 +51,8 @@ packages:
variants: +mpi
libfabric:
variants: fabrics=gni
mercury:
variants: +udreg
all:
providers:
mpi: [mpich]
......@@ -23,6 +23,8 @@ JOBDIR=$PWD/mochi-regression-job-$$
# scratch area to clone and build things
mkdir -p $SANDBOX
cp packages.yaml $SANDBOX/
cp spack-libfabric-1.7.0.patch $SANDBOX/
cp spack-mercury-udreg-variant.patch $SANDBOX/
# scratch area for job submission
mkdir -p $JOBDIR
......@@ -41,6 +43,8 @@ git clone https://github.com/pdlfs/mercury-runner.git
echo "=== BUILD SPACK PACKAGES AND LOAD ==="
cd $SANDBOX/spack
git checkout carns/dev-shell-detection
patch -p1 < $SANDBOX/spack-libfabric-1.7.0.patch
patch -p1 < $SANDBOX/spack-mercury-udreg-variant.patch
. $SANDBOX/spack/share/spack/setup-env.sh
# put packages file in place in SPACK_ROOT to set our preferences for building
# Mochi stack
......
diff --git a/var/spack/repos/builtin/packages/mercury/package.py b/var/spack/repos/builtin/packages/mercury/package.py
index e69d19d4b..a2a65a4a4 100644
--- a/var/spack/repos/builtin/packages/mercury/package.py
+++ b/var/spack/repos/builtin/packages/mercury/package.py
@@ -25,7 +25,6 @@ class Mercury(CMakePackage):
depends_on('cci@master', when='+cci', type=('build', 'link', 'run'))
depends_on('libfabric', when='+fabric', type=('build', 'link', 'run'))
- depends_on('libfabric@develop', when='+fabric platform=cray', type=('build', 'link', 'run'))
depends_on('bmi', when='+bmi', type=('build', 'link', 'run'))
depends_on('openpa', type=('build', 'link', 'run'))
diff --git a/var/spack/repos/builtin/packages/mercury/package.py b/var/spack/repos/builtin/packages/mercury/package.py
index e69d19d4b..fd4bcdfdc 100644
--- a/var/spack/repos/builtin/packages/mercury/package.py
+++ b/var/spack/repos/builtin/packages/mercury/package.py
@@ -22,6 +22,11 @@ class Mercury(CMakePackage):
variant('selfforward', default=True,
description='Mercury will short-circuit operations' +
' by forwarding to itself when possible')
+# NOTE: the 'udreg' variant requires that the MPICH_GNI_NDREG_ENTRIES=1024
+# environment variable be set at run time to avoid conflicts with
+# Cray-MPICH if libfabric and MPI are used at the same time
+ variant('udreg', default=False,
+ description='Enable udreg on supported Cray platforms')
depends_on('cci@master', when='+cci', type=('build', 'link', 'run'))
depends_on('libfabric', when='+fabric', type=('build', 'link', 'run'))
@@ -53,4 +58,9 @@ class Mercury(CMakePackage):
else:
args.extend(['-DMERCURY_USE_SELF_FORWARD=OFF'])
+ if (self.spec.variants['udreg'].value):
+ args.extend(['-DNA_OFI_GNI_USE_UDREG=ON'])
+ else:
+ args.extend(['-DNA_OFI_GNI_USE_UDREG=OFF'])
+
return args
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment