Commit c931515c authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

added Bebop files

parent c52fd545
Bebop
=====
Programming environment
-----------------------
The Bebop machine has many compilers and MPI implementations, not all
of which work correctly with Mochi. We have found `gcc@8.2.0` with
the `intel-mpi` package to be the most reliable way of working with
Mochi on this platform. You can work with these packages by typping:
```
$ module load gcc/8.2.0-g7hppkz
```
This command will automatically change the compiler to `gcc@8.2.0`
and change the `intel-mpi` package to the appropriate one.
Networking
----------
Bebop uses an Intel Omnipath fabric. The corresponding transport in
Mercury is `ofi+psm2`, using the `+psm2` variant in `libfabric`.
Job management
--------------
Bebop uses the Slurm workload manager. `job.sbatch` is an example
of job file. Please modify the header (lines starting with `#SBATCH`)
to use your project allocation and set relevant parameters. You can
refer to [the documentation here](https://www.lcrc.anl.gov/systems/resources/bebop/)
for more information on how to configure such a job script.
Once modified, the job script may be submitted as follows.
```
$ sbatch ./job.sbatch
```
#!/bin/bash
#SBATCH -N 4
#SBATCH -p bdwall
#SBATCH -A MY_PROJECT
#SBATCH --time=30:00
#SBATCH --export=ALL
set -eu
export PSM2_MULTI_EP=1 # prevents conflict with MPI using PSM2
export FI_PSM2_DISCONNECT=1 # enables reconnection
echo "Activating env"
. $HOME/spack/share/spack/setup-env.sh
module load gcc/8.2.0-g7hppkz
spack env activate myenv
echo "Starting application"
srun -N 2 -n 4 ...
spack:
specs:
- mochi-margo%gcc@8.2.0
concretization: together
compilers:
- compiler:
paths:
cc: /blues/gpfs/software/centos7/spack/opt/spack/linux-centos7-x86_64/gcc-8.1.0/gcc-8.2.0-g7hppkz/bin/gcc
cxx: /blues/gpfs/software/centos7/spack/opt/spack/linux-centos7-x86_64/gcc-8.1.0/gcc-8.2.0-g7hppkz/bin/g++
f77: /blues/gpfs/software/centos7/spack/opt/spack/linux-centos7-x86_64/gcc-8.1.0/gcc-8.2.0-g7hppkz/bin/gfortran
fc: /blues/gpfs/software/centos7/spack/opt/spack/linux-centos7-x86_64/gcc-8.1.0/gcc-8.2.0-g7hppkz/bin/gfortran
operating_system: centos7
target: x86_64
modules:
- gcc/8.2.0-g7hppkz
- intel-mpi/2018.4.274-ozfo327
environment: {}
extra_rpaths: []
flags: {}
spec: gcc@8.2.0
repos:
- path/to/sds-repo
packages:
all:
providers:
mpi: [ intel-mpi ]
compiler: [ gcc@8.2.0 ]
autoconf:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: autoconf@2.69 arch=linux-centos7-x86_64
prefix: /usr
automake:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: automake@1.13.4 arch=linux-centos7-x86_64
prefix: /usr
binutils:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: binutils@2.27 arch=linux-centos7-x86_64
prefix: /usr
bison:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: bison@3.0.4 arch=linux-centos7-x86_64
prefix: /usr
bzip2:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: bzip2@1.0.6 arch=linux-centos7-x86_64
prefix: /usr
cmake:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: cmake@3.14.2 arch=linux-centos7-x86_64
modules:
- cmake/3.14.2-gvwazz3
coreutils:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: coreutils@8.25 arch=linux-centos7-x86_64
prefix: /usr
findutils:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: findutils@4.5.11 arch=linux-centos7-x86_64
prefix: /usr
flex:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: flex@2.5.37 arch=linux-centos7-x86_64
prefix: /usr
gettext:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: gettext@0.19.8.1 arch=linux-centos7-x86_64
prefix: /usr
git:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: git@1.8.3.1 arch=linux-centos7-x86_64
prefix: /usr
hdf5:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: hdf5@1.10.5 arch=linux-centos7-x86_64
modules:
- hdf5/1.10.5-vozfsah
intel-mpi:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: intel-mpi@2018.4.274 arch=linux-centos7-x86_64
modules:
- intel-mpi/2018.4.274-ozfo327
libfabric:
variants: fabrics=psm2,sockets
version: []
target: []
compiler: []
buildable: true
providers: {}
libtool:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: libtool@2.4.2 arch=linux-centos7-x86_64
prefix: /usr
m4:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: m4@1.4.16 arch=linux-centos7-x86_64
prefix: /usr
mercury:
variants: ~boostsys
version: []
target: []
compiler: []
buildable: true
providers: {}
numactl:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: numactl@2.0.12 arch=linux-centos7-x86_64
modules:
- numactl/2.0.12-355ef36
openssl:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: openssl@1.0.2k arch=linux-centos7-x86_64
prefix: /usr
perl:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: perl@5.16.3 arch=linux-centos7-x86_64
prefix: /usr
pkg-config:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: pkg-config@0.27.1 arch=linux-centos7-x86_64
prefix: /usr
tar:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: tar@1.16 arch=linux-centos7-x86_64
prefix: /usr
texinfo:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: texinfo@5.1 arch=linux-centos7-x86_64
prefix: /usr
xz:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: xz@5.2.2 arch=linux-centos7-x86_64
prefix: /usr
zlib:
buildable: false
version: []
target: []
compiler: []
providers: {}
externals:
- spec: zlib@1.2.7 arch=linux-centos7-x86_64
prefix: /usr
view: true
...@@ -3,4 +3,116 @@ Platform configurations for Mochi ...@@ -3,4 +3,116 @@ Platform configurations for Mochi
This repository provides Spack configuration files, example job scripts, and This repository provides Spack configuration files, example job scripts, and
notes about building and running Mochi-based codes on various platforms. notes about building and running Mochi-based codes on various platforms.
Please refer to your platform of interest for more information. Please refer to your platform of interest for more information.
\ No newline at end of file
Using spack.yaml files
----------------------
Each platform subdirectory in this repository provides a `spack.yaml` file.
Such a file is meant to fully describe a Spack environment, including some system-provided
packages and compilers. It does so independently of any `compilers.yaml` or `packages.yaml`
files installed in `~/.spack`, hence preventing as much as possible the possible interference
with user-specific spack configurations.
You may use `spack.yaml` files to create a
[Spack environment](https://spack.readthedocs.io/en/latest/environments.html) in
which Mochi packages will be installed.
If you don't have Spack installed on your platform, clone it and set it up as follows.
```
$ git clone https://github.com/spack/spack.git
$ . spack/share/spack/setup-env.sh
```
You may want to put the second line in your `.bashrc` file so that it
gets executed when opening a terminal.
You will then need to clone `sds-repo`, which contains the Mochi packages.
```
$ git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git
$ spack repo add sds-repo
```
Now clone the present repository and `cd` into the subdirectory relevant
to your platform. For example:
```
$ git clone https://xgitlab.cels.anl.gov/sds/experiments/platform-configurations.git
$ cd platform-configurations/ANL/Bebop
```
Edit the path to `sds-repo` in the `repos` field of the `spack.yaml` file to
match your installation.
Then, execute the following command
(changing _myenv_ into an appropriate name for your environment).
```
$ spack env create myenv spack.yaml
```
Change to a directory outside the `platform-configurations` folders
and activate the environment as follows.
```
$ spack env activate myenv
```
You may now add specs to your environment. For instance if you want
to install Margo, execute the following command.
```
$ spack add mochi-margo
```
If the `spack.yaml` file provides multiple compilers and you want
to use another than the default one, specify the compiler explicitely,
for example:
```
$ spack add mochi-margo %gcc@8.2.0
```
Note that the `spack.yaml` file you used may already have a spec
added as an example (usually `mochi-margo`). You can remove it as
follows.
```
$ spack rm mochi-margo
```
Once you have added the specs you need in your environment, install
everything by executing the following command.
```
$ spack install
```
You may add more specs later on. For more information on how to manage
Spack environments, please refer to the Spack documentation.
Contributing to this repository
-------------------------------
Should you want to contribute a `spack.yaml` for a particular machine,
please submit a merge request with it, and ensure the following.
* The `spack.yaml` file should contain the compiler(s) that have been tested
to be working with Mochi packages.
* The `spack.yaml` file should try to list system-provided packages,
in particular packages used for building (`cmake`, `autoconf`, etc.),
and relevant system-provided MPI implementations.
* The `spack.yaml` file should contain the relevant variants for packages,
in particular the transport methods to use with `libfabric`.
* The path to the `spack.yaml` file should be of the form
`<institution>/<platform>/spack.yaml`.
* Please make sure that your `spack.yaml` is a reliable way to work with
Mochi on the target platform, other people will rely on it!
You can also contribute changes to existing `spack.yaml` files, in particular
to add working compilers, system packages, etc. As always, please test that
new setups work before creating a merge request.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment