[feature] Restructuring bandit control.

Adds BwCR, uses submodule HBandit, changes daemon CLI interface,
adds CI steps, largely drafts reference measurement controller
wrapper, adds GLPK bindings, drafts tutorial notebooks.

.gitlab-ci.yml

+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+
 stages:
   - source
   - lib
@@ -25,8 +28,7 @@ ormolu:
         }
       ' --run bash <<< '
         RETURN=0
-        cd hsnrm
-        for F in $(fd -e hs); do
+        for F in $(fd -E hsnrm/hbandit -e hs); do
           ormolu -o -XTypeApplications -m check $F
           if [ $? -ne 0 ]; then
             echo "[!] $F does not pass ormolu haskell format check." >&2
@@ -42,7 +44,11 @@ hlint:
     - kvm
     - nix
   script:
-    - nix-shell --pure -p '(import ./.).hlint' --run bash <<< 'hlint hsnrm/ --hint=./.hlint.yaml'
+    - |
+      nix-shell --pure -p '(import ./.).hlint' --run bash <<< '
+        hlint hsnrm/nrm/ --hint=./.hlint.yaml
+        hlint hsnrm/bin/ --hint=./.hlint.yaml
+      '
 
 black:
   stage: source
@@ -65,8 +71,7 @@ clang-format:
     - |
       nix-shell --pure -p '(import ./.).fd' '(import ./.).clang-tools' --run bash <<< '
         RETURN=0
-        cd libnrm
-        for F in $(fd -e c); do
+        for F in $(fd -E hsnrm/hbandit -e c); do
           clang-format < $F | cmp -s $F -
           if [ $? -ne 0 ]; then
             echo "[!] $F does not pass clang-format format check." >&2
@@ -76,7 +81,24 @@ clang-format:
         if [ $RETURN -ne 0 ]; then exit 1; fi
       '
 
-
+dhall-format:
+  stage: source
+  tags:
+    - kvm
+    - nix
+  script:
+    - |
+      nix-shell --pure -p '(import ./.).fd' '(import ./.).haskellPackages.dhall' --run bash <<< '
+        RETURN=0
+        for F in $(fd -E hsnrm/hbandit -e dhall); do
+          dhall format < $F | cmp -s $F -
+          if [ $? -ne 0 ]; then
+            echo "[!] $F does not pass clang-format format check." >&2
+            RETURN=1
+          fi
+        done
+        if [ $RETURN -ne 0 ]; then exit 1; fi
+      '
 
 nix-build-nrm.so:
   stage: lib
@@ -110,13 +132,13 @@ check-vendored-documentation:
     - nix
   script:
     - |
-      nix-shell dev/default.nix -A hsnrm-hack --run bash <<< '
+      nix-shell dev/default.nix -A hack --run bash <<< '
         echo "check: is vendored documentation up to date?"
-        cd hsnrm
-        cabal v2-haddock nrm.so --haddock-hyperlink-source --builddir=../doc_tmp
-        diff -r ../doc_tmp/build/x86_64-linux/ghc-8.6.5/hsnrm-1.0.0/x/nrm.so/doc/html/hsnrm/nrm.so ../doc/nrm.so/haddocks
-        rm -rf ../doc_tmp
-      '
+        cp -r doc/ doc2
+        ./shake doc
+        ./shake notebooks
+        diff -r doc doc2
+        '
 
 nix-build-resources:
   stage: codegen
@@ -229,6 +251,7 @@ readthedocs:
   when: on_success
   only:
     - master
+    - staging
   tags:
     - kvm
     - nix

.gitmodules

+[submodule "hsnrm/hbandit"]
+	path = hsnrm/hbandit
+	url = ../hbandit.git

README.md

@@ -14,7 +14,7 @@ libnrm-instrumented applications can also be installed using `nix-env -f.  -iA s
 
 ### Development
 
-from a local clone:
+from a local clone, *with submodules initialized and updated*:
 
 ##### read internal haddock documentation: 
 
@@ -53,3 +53,21 @@ your latest local commit.
 ##### install NRM to a (Nix-enabled) remote system user environment:
 
 `./deploy nrm user@host`
+
+##### jupyter notebooks:
+
+Tutorial Notebooks:
+* Unexecuted: [`notebooks/raw`](notebooks/raw)
+* Executed: [`notebooks/executed`](notebooks/executed)
+
+Exploratory Notebooks:
+* Unexecuted: [`notebooks/exploratory`](notebooks/exploratory)
+
+* Running the notebooks in development mode:
+
+```
+./shake build
+./shake client
+./shake pyclient
+nix-shell --run "jupyter-notebook <notebook file>"
+```

dev/cabal/bin.dhall

@@ -4,112 +4,132 @@ let types = ./dhall-to-cabal/types.dhall
 
 let common = ./common.dhall
 
-in  \(ghcPath:Text)  -> \(ghcNumericVersion:Text) -> prelude.defaults.Package
-    ⫽ { name =
-          "hsnrm"
-      , version =
-          prelude.v "1.0.0"
-      , author =
-          "Valentin Reis"
-      , build-type =
-          Some types.BuildType.Simple
-      , cabal-version =
-          prelude.v "2.0"
-      , category =
-          "tools"
-      , description =
-          "The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
-      , executables =
-          [ { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "Export.hs"
-                    , build-depends =
-                        [common.nobound "nrmlib", common.deps.base, common.deps.protolude , common.deps.enclosed-exceptions ]
-                    , hs-source-dirs =
-                        [ "bin" ]
-                    }
-                  ⫽ common.copts
-                    [ "-Wmissed-specialisations"
-                    , "-Wall-missed-specialisations"
-                    , "-fPIC"
-                    , "-shared"
-                    , "-no-hs-main"
-                    , "-dynamic"
-            , "-lHSrts-ghc" ++ ghcNumericVersion
-            , "-L" ++ ghcPath ++ "/lib/ghc-" ++ ghcNumericVersion ++ "/rts/"
-                    ]
-            , name =
-                "nrm.so"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "Hnrm.hs"
-                    , build-depends =
-                        [common.nobound "nrmlib"]
-                    , hs-source-dirs =
-                        [ "bin" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Hnrm" ]
-            , name =
-                "nrmstatic"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "bin/Hnrm.hs"
-                    , build-depends =
-                        [ common.nobound "nrmlib" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Hnrm" ]
-            , name =
-                "nrm"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "bin/Hnrmd.hs"
-                    , build-depends =
-                        [ common.nobound "nrmlib" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Hnrmd" ]
-            , name =
-                "fake-daemon"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "bin/Codegen.hs"
-                    , build-depends =
-                        [ common.nobound "nrmlib" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Codegen" ]
-            , name =
-                "codegen"
-            }
-          ]
-      , extra-source-files =
-          [ "ChangeLog.md" ]
-      , license =
-          types.License.BSD3
-      , license-files =
-          [] : List Text
-      , maintainer =
-          "fre@freux.fr"
-      , source-repos =
-          [   prelude.defaults.SourceRepo
-            ⫽ { type =
-                  Some types.RepoType.Git
-              , location =
-                  Some "https://xgitlab.cels.anl.gov/vreis/hsnrm.git"
+in    λ ( ghcPath
+        : Text
+        )
+    → λ(ghcNumericVersion : Text)
+    →   prelude.defaults.Package
+      ⫽ { name =
+            "hsnrm"
+        , version =
+            prelude.v "1.0.0"
+        , author =
+            "Valentin Reis"
+        , build-type =
+            Some types.BuildType.Simple
+        , cabal-version =
+            prelude.v "2.0"
+        , category =
+            "tools"
+        , description =
+            "The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
+        , executables =
+            [ { executable =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Executable
+                    ⫽ { main-is =
+                          "Export.hs"
+                      , build-depends =
+                          [ common.nobound "nrmlib"
+                          , common.deps.base
+                          , common.deps.protolude
+                          , common.deps.enclosed-exceptions
+                          ]
+                      , hs-source-dirs =
+                          [ "bin" ]
+                      , c-sources =
+                          [ "glpk/glpk.c" ]
+                      , extra-libraries =
+                          [ "glpk" ]
+                      }
+                    ⫽ common.copts
+                      [ "-Wmissed-specialisations"
+                      , "-Wall-missed-specialisations"
+                      , "-fPIC"
+                      , "-shared"
+                      , "-no-hs-main"
+                      , "-dynamic"
+                      , "-lHSrts-ghc" ++ ghcNumericVersion
+                      ,     "-L"
+                        ++  ghcPath
+                        ++  "/lib/ghc-"
+                        ++  ghcNumericVersion
+                        ++  "/rts/"
+                      ]
+              , name =
+                  "nrm.so"
               }
-          ]
-      , synopsis =
-          "hsnrm"
-      }
+            , { executable =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Executable
+                    ⫽ { main-is =
+                          "Hnrm.hs"
+                      , build-depends =
+                          [ common.nobound "nrmlib" ]
+                      , hs-source-dirs =
+                          [ "bin", "src", "hbandit/src" ]
+                      , c-sources =
+                          [ "glpk/glpk.c" ]
+                      , extra-libraries =
+                          [ "glpk" ]
+                      }
+                    ⫽ common.copts [ "-main-is", "Hnrm" ]
+              , name =
+                  "nrmstatic"
+              }
+            , { executable =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Executable
+                    ⫽ { main-is =
+                          "bin/Hnrm.hs"
+                      , build-depends =
+                          [ common.nobound "nrmlib" ]
+                      }
+                    ⫽ common.copts [ "-main-is", "Hnrm" ]
+              , name =
+                  "nrm"
+              }
+            , { executable =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Executable
+                    ⫽ { main-is =
+                          "bin/Hnrmd.hs"
+                      , build-depends =
+                          [ common.nobound "nrmlib" ]
+                      }
+                    ⫽ common.copts [ "-main-is", "Hnrmd" ]
+              , name =
+                  "fake-daemon"
+              }
+            , { executable =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Executable
+                    ⫽ { main-is =
+                          "bin/Codegen.hs"
+                      , build-depends =
+                          [ common.nobound "nrmlib" ]
+                      }
+                    ⫽ common.copts [ "-main-is", "Codegen" ]
+              , name =
+                  "codegen"
+              }
+            ]
+        , extra-source-files =
+            [ "ChangeLog.md" ]
+        , license =
+            types.License.BSD3
+        , license-files =
+            [] : List Text
+        , maintainer =
+            "fre@freux.fr"
+        , source-repos =
+            [   prelude.defaults.SourceRepo
+              ⫽ { type =
+                    Some types.RepoType.Git
+                , location =
+                    Some "https://xgitlab.cels.anl.gov/vreis/hsnrm.git"
+                }
+            ]
+        , synopsis =
+            "hsnrm"
+        }

dev/cabal/common.dhall

@@ -17,6 +17,7 @@ let defexts =
       , types.Extension.OverloadedStrings True
       , types.Extension.ViewPatterns True
       , types.Extension.DeriveFunctor True
+      , types.Extension.DeriveTraversable True
       , types.Extension.TypeFamilies True
       , types.Extension.DeriveAnyClass True
       , types.Extension.DeriveGeneric True
@@ -110,6 +111,8 @@ let deps =
           nobound "hxt-xpath"
       , recursion-schemes =
           nobound "recursion-schemes"
+      , binary =
+          nobound "binary"
       , refined =
           nobound "refined"
       , generic-data =
@@ -250,6 +253,7 @@ let modules =
       , "NRM.Types.DownstreamCmd"
       , "NRM.Types.DownstreamCmdID"
       , "NRM.Types.State"
+      , "NRM.Types.MemBuffer"
       , "NRM.Types.UpstreamClient"
       , "NRM.Types.DownstreamClient"
       , "NRM.Classes.Messaging"
@@ -261,6 +265,8 @@ let modules =
       , "NRM.Orphans.UUID"
       , "NRM.Orphans.Dhall"
       , "NRM.Orphans.NonEmpty"
+      , "NRM.Orphans.ZeroOne"
+      , "NRM.Orphans.Refined"
       , "NRM.Types.Messaging.DownstreamEvent"
       , "NRM.Types.Messaging.UpstreamPub"
       , "NRM.Types.Messaging.UpstreamReq"
@@ -289,9 +295,33 @@ let extramodules =
       , "CPD.Values"
       , "CPD.Utils"
       , "CPD.Text"
+      , "Control.Monad.LPMonad.Internal"
+      , "Control.Monad.LPMonad.Supply.Class"
+      , "Control.Monad.LPMonad.Supply"
+      , "Control.Monad.LPMonad"
+      , "Data.LinearProgram"
+      , "Data.LinearProgram.Common"
+      , "Data.LinearProgram.GLPK"
+      , "Data.LinearProgram.GLPK.Common"
+      , "Data.LinearProgram.GLPK.Internal"
+      , "Data.LinearProgram.GLPK.IO.Internal"
+      , "Data.LinearProgram.GLPK.IO"
+      , "Data.LinearProgram.GLPK.Solver"
+      , "Data.LinearProgram.GLPK.Types"
+      , "Data.LinearProgram.LinExpr"
+      , "Data.LinearProgram.Spec"
+      , "Data.LinearProgram.Types"
+      , "Algebra.Classes"
       ]
 
-let banditmodules = [ "Bandit.Class", "Bandit.Exp3", "Bandit.EpsGreedy", "Bandit.Util" ]
+let banditmodules =
+      [ "HBandit.Class"
+      , "HBandit.Exp3"
+      , "HBandit.EpsGreedy"
+      , "HBandit.Util"
+      , "HBandit.Types"
+      , "HBandit.BwCR"
+      ]
 
 let allmodules = modules # extramodules # banditmodules
 
@@ -312,6 +342,7 @@ let libdep =
       , deps.generic-lens
       , deps.data-msgpack
       , deps.containers
+      , deps.binary
       , deps.unordered-containers
       , deps.mtl
       , deps.aeson
@@ -355,7 +386,8 @@ let libdep =
 
 in  { defexts =
         defexts
-    ,deps = deps
+    , deps =
+        deps
     , libdep =
         libdep
     , banditmodules =

dev/cabal/dev.dhall

@@ -4,142 +4,167 @@ let types = ./dhall-to-cabal/types.dhall
 
 let common = ./common.dhall
 
-in  \(ghcPath:Text)  -> \(ghcNumericVersion:Text) ->   prelude.defaults.Package
-    ⫽ { name =
-          "hsnrm"
-      , version =
-          prelude.v "1.0.0"
-      , author =
-          "Valentin Reis"
-      , build-type =
-          Some types.BuildType.Simple
-      , cabal-version =
-          prelude.v "2.0"
-      , category =
-          "tools"
-      , description =
-          "The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
-      , sub-libraries =
-          [ { library =
-                  λ(config : types.Config)
-                →   prelude.defaults.Library
-                  ⫽ { build-depends =
-                        common.libdep
-                    , hs-source-dirs =
-                        [ "nrm", "bin" ]
-                    , exposed-modules =
-                        common.allmodules
-                    }
-                  ⫽ common.copts ([] : List Text)
-            , name =
-                "nrmlib"
-            }
-          , { library =
-                  λ(config : types.Config)
-                →   prelude.defaults.Library
-                  ⫽ { build-depends =
-                        common.libdep
-                    , hs-source-dirs =
-                        [ "nrm" ]
-                    , exposed-modules =
-                        common.banditmodules
-                    }
-                  ⫽ common.copts ([] : List Text)
-            , name =
-                "banditlib"
-            }
-          ]
-      , executables =
-          [ { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "Export.hs"
-                    , build-depends =
-                        common.libdep
-                    , hs-source-dirs =
-                        [ "bin", "nrm" ]
-                    , other-modules =
-                        common.allmodules
-                    }
-                  ⫽ common.copts
-                    [ "-fPIC"
-                    , "-shared"
-                    , "-no-hs-main"
-                    , "-dynamic"
-                    ]
-            , name =
-                "nrm.so"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "Hnrm.hs"
-                    , build-depends =
-                        common.libdep
-                    , hs-source-dirs =
-                        [ "bin", "nrm" ]
-                    , other-modules =
-                        common.allmodules
-                    }
-                  ⫽ common.copts [ "-main-is", "Hnrm" ]
-            , name =
-                "nrmstatic"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "bin/Hnrm.hs"
-                    , build-depends =
-                        [ common.nobound "nrmlib" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Hnrm" ]
-            , name =
-                "nrm"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "bin/Hnrmd.hs"
-                    , build-depends =
-                        [ common.nobound "nrmlib" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Hnrmd" ]
-            , name =
-                "nrmddep"
-            }
-          , { executable =
-                  λ(config : types.Config)
-                →   prelude.defaults.Executable
-                  ⫽ { main-is =
-                        "bin/Codegen.hs"
-                    , build-depends =
-                        [ common.nobound "nrmlib" ]
-                    }
-                  ⫽ common.copts [ "-main-is", "Codegen" ]
-            , name =
-                "codegen"
-            }
-          ]
-      , extra-source-files =
-          [ "ChangeLog.md" ]
-      , license =
-          types.License.BSD3
-      , license-files =
-          [] : List Text
-      , maintainer =
-          "fre@freux.fr"
-      , source-repos =
-          [   prelude.defaults.SourceRepo
-            ⫽ { type =
-                  Some types.RepoType.Git
-              , location =
-                  Some "https://xgitlab.cels.anl.gov/vreis/hsnrm.git"
+in    λ ( ghcPath
+        : Text
+        )
+    → λ(ghcNumericVersion : Text)
+    →   prelude.defaults.Package
+      ⫽ { name =
+            "hsnrm"
+        , version =
+            prelude.v "1.0.0"
+        , author =
+            "Valentin Reis"
+        , build-type =
+            Some types.BuildType.Simple
+        , cabal-version =
+            prelude.v "2.0"
+        , category =
+            "tools"
+        , description =
+            "The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
+        , sub-libraries =
+            [ { library =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Library
+                    ⫽ { build-depends =
+                          common.libdep
+                      , hs-source-dirs =
+                          [ "nrm", "bin", "hbandit/src" ]
+                      , exposed-modules =
+                          common.allmodules
+                      }
+                    ⫽ common.copts ([] : List Text)
+              , name =
+                  "nrmlib"
               }
-          ]
-      , synopsis =
-          "hsnrm"
-      }
+            , { library =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Library
+                    ⫽ { build-depends =
+                          common.libdep
+                      , hs-source-dirs =
+                          [ "nrm" ]
+                      , exposed-modules =
+                          common.banditmodules
+                      }
+                    ⫽ common.copts ([] : List Text)
+              , name =
+                  "banditlib"
+              }
+            ]
+        , executables =
+            [ { executable =
+                    λ(config : types.Config)
+                  →   prelude.defaults.Executable
+                    ⫽ { main-is =
+                          "Export.hs"
+                      , build-depends =
+                          common.libdep
+                      , hs-source-dirs =
+                          [ "bin", "nrm", "hbandit/src" ]