Merge branch '47-readthedocs' into 'master'

Resolve "readthedocs"

Closes #47

See merge request !80

.gitignore

@@ -49,3 +49,7 @@ venv/
 
 # integration tests
 _output
+
+# documentation
+docs/_build
+docs/manifest.rst

.gitlab-ci.yml

@@ -8,10 +8,12 @@ stages:
   - build
   - test
   - applications
+  - docs
   - quality
 
 include:
   - https://xgitlab.cels.anl.gov/argo/argopkgs/raw/master/gitlab-ci/argonix.yml
+  - https://xgitlab.cels.anl.gov/argo/argopkgs/raw/master/gitlab-ci/nrm.yml
   - https://xgitlab.cels.anl.gov/argo/argopkgs/raw/master/gitlab-ci/components.yml
   - https://xgitlab.cels.anl.gov/argo/argopkgs/raw/master/gitlab-ci/integration.yml
   - https://xgitlab.cels.anl.gov/argo/argopkgs/raw/master/gitlab-ci/applications.yml

.readthedocs.yml

+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  builder: html
+  configuration: docs/conf.py
+
+python:
+  install:
+    - requirements: docs/requirements.txt

default.nix

@@ -4,6 +4,8 @@
 rec {
   nrm = pkgs.nrm;
   hack = nrm.overrideAttrs (old:{
-    buildInputs = old.buildInputs ++ [pkgs.pythonPackages.flake8];
+    buildInputs = old.buildInputs ++ [
+      pkgs.pythonPackages.flake8
+      pkgs.pythonPackages.sphinx ];
   });
 }

docs/.gitignore

+_*

docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = NRM
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+

docs/conf.py

+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+# import os
+# import sys
+import subprocess
+
+subprocess.call("sed '/^# /! s/^/  /g' ../nrm/schemas/manifest.yml "
+                "| sed '/^# / s/# //g' > manifest.rst", shell=True)
+
+
+# -- Project information -----------------------------------------------------
+
+project = u'NRM'
+copyright = u'2019, Argonne National Laboratory'
+author = u'Argo team, Argonne National Laboratory'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'NRMdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'NRM.tex', u'NRM Documentation',
+     u'ANL', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'nrm', u'NRM Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'NRM', u'NRM Documentation',
+     author, 'NRM', 'One line description of project.',
+     'Miscellaneous'),
+]

docs/index.rst

+Welcome to NRM's documentation!
+===============================
+
+If you know about NRM and are just looking to get it to run on your
+application, please visit the :doc:`quickstart <quickstart>` guide.
+
+This documentation is technical. For a high-level overview of NRM, please
+refer to the Argo website_.
+
+The Node Resource Manager(NRM) is a node-local userspace client-server daemon
+for managing your scientific applications. It runs the various tasks that
+compose an application in resource-constrained slices, monitors
+performance, power use and application progress, and arbitrates resources at
+the node level, among which CPU Cores, NUMA Nodes, and Power budgets.
+
+There are two user software components shipped with NRM itself: the `nrm`
+command-line client and the `nrmd` daemon. Additionally, NRM ships with the
+`libnrm` application instrumentation library, to be used for progress
+monitoring. The following diagram describes this architecture:
+
+ .. image:: nrm.svg
+
+Note that the container runtime used by NRM to allocate slices is a
+system-installed dependency, regardless of whether Argo NodeOS or Singularity is used.
+
+The :doc:`quickstart <quickstart>` guide describes the use of `nrm` and `nrmd`.
+An important part of running tasks with `nrm` is the application :doc:`manifest
+<manifest>`, which you probably want to read about too.  Finally, please refer to the :doc:`libnrm <libnrm>` guide for
+application istrumentation.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   quickstart
+   manifest
+   libnrm
+
+Indices and tables
+==================
+
+* :ref:`search`
+
+ .. _website: https://www.mcs.anl.gov/research/projects/argo/overview/nrm/

docs/libnrm.rst

+======
+Libnrm
+======
+
+.. highlight:: bash
+
+Welcome to the libnrm guide. This document will help you in your use of the
+C/C++ and Fortran interface for NRM.
+
+Install
+=======
+
+The libnrm code can be installed for source::
+
+ git clone https://xgitlab.cels.anl.gov/argo/libnrm.git
+ cd libnrm
+ ./autogen.sh
+ ./configure
+ make && make install
+
+API
+===
+
+**nrm_ctxt_create(void)**
+
+This function creates a context and returns it.
+
+**nrm_init(struct nrm_context *, const char *)**
+
+This funciton starts the connection with NRM and gives all the application's
+communications a label.
+
+**`nrm_send_progress(struct nrm_context *, unsigned long progress)**
+
+This funciton sends the progress of the application to NRM.
+
+**nrm_fini(struct nrm_context *)**
+
+This function close the connection to NRM.
+
+**nrm_ctxt_delete(struct nrm_context *)**
+
+This functions deletes the NRM context.
+
+Using libnrm in your C/ C++ application
+=======================================
+.. highlight:: C
+
+The best way to understand how to make your application report progress to NRM
+is to use an example::
+
+ # include <stdio.h>
+
+ int main()
+ {
+   int i;
+
+   printf("hello\n")
+
+   for (i=1; j<4; i++){
+     printf("number %i\n",i);
+   }
+
+   printf("done!");
+   return 0;
+ }
+
+To make this code report progress to NRM, we need to:
+
+- include the library in the same way you include your other dependencies,
+- declare a nrm_context structure,
+- initialize your NRM context,
+- report progress to NRM at one point in your code,
+- close the connection and delete your NRM context,
+
+using the functions from the API. We end up with something like this::
+
+ # include <stdio.h>
+ # include <nrm.h>
+
+ int main()
+ {
+   int i;
+   struct nrm_context context;
+
+   printf("hello\n")
+   nrm_init(&context, "example");
+
+   for (i=1; j<4; i++){
+     printf("number %i\n",i);
+     nrm_send_progress(&context, 1);
+   }
+
+   printf("done!");
+   nrm_fini(&context);
+   return 0;
+ }
+
+Using libnrm in your Fortran application
+========================================
+.. highlight:: Fortran
+
+The same thing can be done with a Fortran application, using the Fortran
+interface of this library. Let's take a similar example::
+
+ implicit none
+
+ integer i
+
+ print*, "hello"
+
+ do i=1, 4
+   print*, "number", i
+ end do
+
+ print*, "done!"
+
+The functions in the Fortran interface are similar to the ones from the C API,
+only with a `f_` in front.
+To talk to NRM, the code becomes::
+
+ implicit none
+
+ include 'f_nrm.h'
+ include(kind=NRM_PTR) context
+
+ integer rc, i
+
+ print*, "hello"
+ rc = f_nrm_ctxt_create(context);
+ rc = f_nrm_init(context, 'example', len('example'))
+
+ do i=1, 4
+   print*, "number", i
+   progress = 1.0
+   rc = f_nrm_send_progress(context, progress)
+ end do
+
+ print*, "done!"
+ rc = f_nrm_fini(context)
+ rc = f_nrm_ctxt_delete(context)

docs/make.bat

+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=NRM
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/nrm.svg

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/PR-SVG-20010719/DTD/svg10.dtd">
+<svg width="39cm" height="17cm" viewBox="89 69 777 332" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+  <g id="Background">
+    <g>
+      <rect style="fill: #ffffff; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="200" y="120" width="160" height="102" rx="0" ry="0"/>
+      <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="280" y="142.9">
+        <tspan x="280" y="142.9">Manifest</tspan>
+        <tspan x="280" y="158.9">(yaml file)</tspan>
+        <tspan x="280" y="174.9">defines application</tspan>
+        <tspan x="280" y="190.9">monitoring </tspan>
+        <tspan x="280" y="206.9">and slice parameters</tspan>
+      </text>
+    </g>
+    <g>
+      <rect style="fill: #ffffff; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="600" y="220" width="140" height="40" rx="0" ry="0"/>
+      <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="670" y="243.9">
+        <tspan x="670" y="243.9">nrmd (daemon)</tspan>
+      </text>
+    </g>
+    <g>
+      <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="670" y1="179.736" x2="670" y2="210.264"/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="670,172.236 675,182.236 670,179.736 665,182.236 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="670,217.764 665,207.764 670,210.264 675,207.764 "/>
+    </g>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="680" y="200">
+      <tspan x="680" y="200">stream RPC</tspan>
+    </text>
+    <g>
+      <rect style="fill: #ffffff; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="560" y="320" width="220" height="80" rx="0" ry="0"/>
+      <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="670" y="355.9">
+        <tspan x="670" y="355.9">Container Runtime</tspan>
+        <tspan x="670" y="371.9">(Argo NodeOS, Singularity, ..)</tspan>
+      </text>
+    </g>
+    <g>
+      <line style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x1="670" y1="269.736" x2="670" y2="309.444"/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="670,262.236 675,272.236 670,269.736 665,272.236 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="670,316.944 665,306.944 670,309.444 675,306.944 "/>
+    </g>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="680" y="280">
+      <tspan x="680" y="280">slice</tspan>
+      <tspan x="680" y="296">start/monitoring</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="140" y="270">
+      <tspan x="140" y="270">Run</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="100" y="250">
+      <tspan x="100" y="250">User Commands</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="140" y="290">
+      <tspan x="140" y="290">Listen</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="140" y="310">
+      <tspan x="140" y="310">Kill</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="140" y="330">
+      <tspan x="140" y="330">Set power</tspan>
+    </text>
+    <g>
+      <polyline style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" points="229.736,270 380,270 380,90 590.264,90 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="222.236,270 232.236,265 229.736,270 232.236,275 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="597.764,90 587.764,95 590.264,90 587.764,85 "/>
+    </g>
+    <g>
+      <polyline style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" points="600,110 600,110 400,110 400,290 229.736,290 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="222.236,290 232.236,285 229.736,290 232.236,295 "/>
+    </g>
+    <g>
+      <polyline style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" points="220,310 420,310 420,130 590.264,130 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="597.764,130 587.764,135 590.264,130 587.764,125 "/>
+    </g>
+    <g>
+      <polyline style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" points="220,330 440,330 440,150 590.264,150 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="597.764,150 587.764,155 590.264,150 587.764,145 "/>
+    </g>
+    <g>
+      <polyline style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" points="280,222 270,222 270,260.264 "/>
+      <polygon style="fill: #000000; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" fill-rule="evenodd" points="270,267.764 265,257.764 270,260.264 275,257.764 "/>
+    </g>
+    <g>
+      <rect style="fill: #ffffff; fill-opacity: 1; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="600" y="70" width="140" height="100" rx="0" ry="0"/>
+      <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="670" y="123.9">
+        <tspan x="670" y="123.9">nrm (CLI client)</tspan>
+      </text>
+    </g>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="500" y="124">
+      <tspan x="500" y="124">stream</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="500" y="104">
+      <tspan x="500" y="104">stream RPC</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="500" y="144">
+      <tspan x="500" y="144">RPC</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="500" y="164">
+      <tspan x="500" y="164">RPC</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="760" y="220">
+      <tspan x="760" y="220">Upstream API</tspan>
+    </text>
+    <text font-size="12.8" style="fill: #000000; fill-opacity: 1; stroke: none;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="760" y="260">
+      <tspan x="760" y="260">Downstream API</tspan>
+    </text>
+    <rect style="fill: none; stroke-opacity: 1; stroke-width: 2; stroke: #000000" x="90" y="230" width="130" height="110" rx="0" ry="0"/>
+  </g>
+</svg>

docs/quickstart.rst

+==========
+Quickstart
+==========
+
+.. highlight:: bash
+
+Welcome to the quickstart guide for NRM. This document will guide you to get up
+and running with running your computational jobs through the Node Resource
+Manager(NRM).
+
+Install
+=======
+
+Container piece
+---------------
+
+The NRM code now supports mapping slices on both Singularity containers and
+NodeOS compute containers.
+
+NodeOS
+~~~~~~
+
+For NodeOScontainer usage, you need to install our container piece
+on the system. On a production platform, this should be done by a sysadmin. On
+a development platform, this can be acheived with::
+
+ git clone https://xgitlab.cels.anl.gov/argo/containers.git
+ cd containers
+ make install
+
+Singularity
+~~~~~~~~~~~
+
+For local singularity installation, refer to the Singularity_ installation
+page.
+
+NRM
+---
+
+The NRM core components (the `nrmd` daemon and `nrm` client) can be installed
+in multiple ways:
+
+using Spack
+~~~~~~~~~~~
+
+ spack install nrm
+
+using Nix
+~~~~~~~~~
+
+NRM has a Nix package in our local package repository::
+
+ nix-env -f "https://xgitlab.cels.anl.gov/argo/argopkgs/-/archive/master/argopkgs-master.tar.gz" -iA nrm
+
+using Pip
+~~~~~~~~~
+
+You should be able to get NRM and its dependencies on any machine with::
+
+ pip install git+https://xgitlab.cels.anl.gov/argo/nrm.git
+
+And entering the resulting virtual environment with `pipenv shell`.
+
+Setup: Launching the `nrmd` daemon
+==================================
+
+NRM's behavior is controlled by the `nrmd` userspace daemon.  The `nrmd` daemon
+should be launched by the application framework in the background and manages
+the resource arbitration on the machine.
+
+The daemon is launched via `nrmd` and logs its output to `/tmp/nrm_log` by
+default.
+
+`nrm` command-line options
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `nrm` daemon is mainly configured
+through its command-line options.::
+
+  usage: nrmd [-h] [-c FILE] [-d] [-v] [--nrm_log NRM_LOG] [--hwloc HWLOC]
+              [--argo_nodeos_config ARGO_NODEOS_CONFIG] [--perf PERF]
+              [--pmpi_lib PMPI_LIB] [--argo_perf_wrapper ARGO_PERF_WRAPPER]
+              [--singularity SINGULARITY]
+              [--container-runtime {nodeos,singularity}]
+
+  optional arguments:
+    -h, --help            show this help message and exit
+    -c FILE, --configuration FILE
+                          Specify a config json-formatted config file to
+                          override any of the available CLI options. If an
+                          option is actually provided on the command-line, it
+                          overrides its corresponding value from the
+                          configuration file.
+    -d, --print_defaults  Print the default configuration file.
+    -v, --verbose         increase output verbosity
+    --nrm_log NRM_LOG     Main log file. Override default with the NRM_LOG
+                          environment variable
+    --hwloc HWLOC         Path to the hwloc to use. This path can be relative
+                          and makes uses of the $PATH if necessary. Override
+                          default with the HWLOC environment variable.
+    --argo_nodeos_config ARGO_NODEOS_CONFIG
+                          Path to the argo_nodeos_config to use. This path can
+                          be relative and makes uses of the $PATH if necessary.
+                          Override default with the ARGO_NODEOS_CONFIG
+                          environment variable.
+    --perf PERF           Path to the linux perf tool to use. This path can be
+                          relative and makes uses of the $PATH if necessary.
+                          Override default with the PERF environment variable.
+    --pmpi_lib PMPI_LIB   Path to the libnrm PMPI library used for the power
+                          policy. Override default with the PMPI environment
+                          variable.
+    --argo_perf_wrapper ARGO_PERF_WRAPPER
+                          Path to the linux perf tool to use. This path can be
+                          relative and makes uses of the $PATH if necessary.
+                          Override default with the PERFWRAPPER environment
+                          variable.
+    --singularity SINGULARITY
+                          Path to the singularity command. Override default with
+                          the SINGULARITY environment variable.
+    --container-runtime {nodeos,singularity}
+                          Choice of container runtime. Override default with the
+                          ARGO_CONTAINER_RUNTIME environment variable.
+
+Running jobs using `nrm`
+========================
+
+Tasks are configured using a JSON file called a manifest and started using the `nrm`
+command-line utility. Here's an example manifest that allocates two CPUS and
+enables application progress monitoring with a one second rate limit.::
+
+  name: basic
+  version: 0.0.1
+  app:
+    container:
+      cpus: 2
+      mems: 1
+    perfwrapper: true
+    monitoring:
+      ratelimit: 1000000000
+
+This manifest can be used in the following way to launch a command::
+
+ $ nrm run /path/to/manifest.yaml echo "foobar"
+ foobar
+ INFO:nrm:process ended: msg_up_rpc_rep_process_exit(api=u'up_rpc_rep', container_uuid=u'b54f12ed-6418-4b32-b6ab-2dda7503a1c8', status=u'0', type=u'process_exit')
+ INFO:nrm:command ended: msg_up_rpc_rep_process_exit(api=u'up_rpc_rep', container_uuid=u'b54f12ed-6418-4b32-b6ab-2dda7503a1c8', status=u'0', type=u'process_exit')
+
+You have run your first nrm-enabled command. See the :doc:`manifest
+guide <manifest>` for an in-depth description of the manifest file format.
+
+`nrm` command-line options
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `nrm` command-line client can be used for a number of operations::
+
+  usage: nrm [-h] [-v] {run,kill,list,listen,setpower} ...
+
+  positional arguments:
+    {run,kill,list,listen,setpower}
+
+  optional arguments:
+    -h, --help            show this help message and exit
+    -v, --verbose         verbose logging information
+
+Start containerized tasks, using a container specification we refer to as an application :doc:`manifest <manifest>`::
+
+  usage: nrm run [-h] [-u [UCONTAINERNAME]] manifest command ...
+
+  positional arguments:
+    manifest              manifest file to apply
+    command               command to execute
+    args                  command arguments
+
+  optional arguments:
+    -h, --help            show this help message and exit
+    -u [UCONTAINERNAME], --ucontainername [UCONTAINERNAME]
+