Commit d29eeebd authored by Matthieu Dorier's avatar Matthieu Dorier

added Theta files

parent c931515c
Theta
=====
Programming environment
-----------------------
Theta has many compilers and MPI implementations, not all
of which work correctly with Mochi. We found multiple version of gcc that
work with Mochi, and the provided `spack.yaml` file uses the latest version
(`gcc@9.3.0`) with the `cray-mpich` package. You can work with these packages
by typping:
```
$ module swap PrgEnv-intel PrgEnv-gnu
$ module load gcc/9.3.0
```
You may want to `export CRAYPE_LINK_TYPE=dynamic` (e.g. in your `.bashrc` file)
to avoid problems with dynamic libraries when building programs.
Networking
----------
Theta uses a Cray Aries fabric. The corresponding transport in
Mercury is `ofi+gni`, using the `+gni` variant in `libfabric`.
Job management
--------------
Bebop uses the Cobalt workload manager. `job.qsub` is an example
of job file. Please modify the header (lines starting with `#COBALT`)
to use your project allocation and set relevant parameters. You can
refer to [the documentation here](https://www.alcf.anl.gov/support-center/theta/submit-job-theta)
for more information on how to configure such a job script.
Once modified, the job script may be submitted as follows.
```
$ qsub ./job.qsub
```
Note the use of a protection domain (pdomain): these are necessary if
you have multiple applications, started using distinct `aprun` calls,
and you want them to communicate via Mercury. Protection domains are
global, but user-specific (i.e. you won't be able to use a pdomain
setup by someone else, and you won't be able to use the same name
as someone else's pdomain). There is a limited number of pdomains
that can exist simultaneously, and they are not cleaned up when your
job terminates, so please make sure to delete your pdomain at the
end of your job.
The `MPICH_GNI_NDREG_ENTRIES` environment variable should be set
to avoid conflicts between MPI and libfabric.
`HDF5_USE_FILE_LOCKING=FALSE` has been shown to be useful for some
workloads using HDF5 on Lustre.
#!/bin/bash
#COBALT -A MY_PROJECT
#COBALT -n 4
#COBALT -t 0:30:00
#COBALT --mode script
#COBALT -q debug-flat-quad
set -eu
export MPICH_GNI_NDREG_ENTRIES=1024
export HDF5_USE_FILE_LOCKING=FALSE
export CRAYPE_LINK_TYPE=dynamic
PDOMAIN=mypdomain
echo "Setting up spack"
source $HOME/spack/share/spack/setup-env.sh
echo "Activating env"
spack env activate myenv
echo "Setting up protection domain"
apstat -P | grep ${PDOMAIN} || apmgr pdomain -c -u ${PDOMAIN}
echo "Starting up application"
aprun -n 4 -N 2 -cc none -d 32 -p ${PDOMAIN} ...
echo "Destroying protection domain"
apmgr pdomain -r -u ${PDOMAIN}
spack:
specs:
- mochi-margo%gcc@9.3.0
concretization: together
compilers:
- compiler:
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
operating_system: cnl7
target: any
modules:
- PrgEnv-gnu
- gcc/9.3.0
environment: {}
extra_rpaths: []
flags: {}
spec: gcc@9.3.0
repos:
- path/to/sds-repo
packages:
all:
providers:
mpi: [ mpich ]
compiler:
- gcc@9.3.0
target:
- mic_knl
version: []
buildable: true
autoconf:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: autoconf@2.69
prefix: /usr
automake:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: automake@1.13.4
prefix: /usr
binutils:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: binutils@2.31.1
prefix: /usr
bison:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: bison@2.7
prefix: /usr
boost:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: boost@1.64.0 arch=cray-cnl7-mic_knl
modules:
- boost/gnu/1.64.0
bzip2:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: bzip2@1.0.6
prefix: /usr
cmake:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: cmake@3.14.5
modules:
- cmake/3.14.5
coreutils:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: coreutils@8.25
prefix: /usr
findutils:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: findutils@4.5.12
prefix: /usr
flex:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: flex@2.5.37
prefix: /usr
gettext:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: gettext@0.19.2
prefix: /usr
git:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: git@2.12.3
prefix: /usr
hdf5:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: hdf5@1.10.6.1
modules:
- cray-hdf5/1.10.6.1
libfabric:
variants: fabrics=gni
version: []
target: []
compiler: []
buildable: true
providers: {}
libtool:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: libtool@2.4.6
prefix: /usr
libuuid:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: libuuid@2.33.1
prefix: /usr
m4:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: m4@1.4.18
prefix: /usr
mercury:
variants: +udreg ~boostsys
version: []
target: []
compiler: []
buildable: true
providers: {}
mpich:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: mpich@7.7.14 arch=cray-cnl7-mic_knl
modules:
- cray-mpich/7.7.14
ncurses:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: ncurses@6.1
prefix: /usr
openssl:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: openssl@1.1.0i
prefix: /usr
perl:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: perl@5.26.1
prefix: /usr
pkg-config:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: pkg-config@0.29.2
prefix: /usr
tar:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: tar@1.27.1
prefix: /usr
texinfo:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: texinfo@4.13
prefix: /usr
xz:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: xz@5.0.5
prefix: /usr
zlib:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: zlib@1.2.11
prefix: /usr
view: true
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment