Commit 399fff43 authored by Valentin Reis's avatar Valentin Reis
Browse files

[feature] Restructuring bandit control.

Adds BwCR, uses submodule HBandit, changes daemon CLI interface,
adds CI steps, largely drafts reference measurement controller
wrapper, adds GLPK bindings, drafts tutorial notebooks.
parent 1c6c9d9d
Pipeline #9313 failed with stages
in 1 minute and 34 seconds
variables:
GIT_SUBMODULE_STRATEGY: recursive
stages:
- source
- lib
......@@ -25,8 +28,7 @@ ormolu:
}
' --run bash <<< '
RETURN=0
cd hsnrm
for F in $(fd -e hs); do
for F in $(fd -E hsnrm/hbandit -e hs); do
ormolu -o -XTypeApplications -m check $F
if [ $? -ne 0 ]; then
echo "[!] $F does not pass ormolu haskell format check." >&2
......@@ -42,7 +44,11 @@ hlint:
- kvm
- nix
script:
- nix-shell --pure -p '(import ./.).hlint' --run bash <<< 'hlint hsnrm/ --hint=./.hlint.yaml'
- |
nix-shell --pure -p '(import ./.).hlint' --run bash <<< '
hlint hsnrm/nrm/ --hint=./.hlint.yaml
hlint hsnrm/bin/ --hint=./.hlint.yaml
'
black:
stage: source
......@@ -65,8 +71,7 @@ clang-format:
- |
nix-shell --pure -p '(import ./.).fd' '(import ./.).clang-tools' --run bash <<< '
RETURN=0
cd libnrm
for F in $(fd -e c); do
for F in $(fd -E hsnrm/hbandit -e c); do
clang-format < $F | cmp -s $F -
if [ $? -ne 0 ]; then
echo "[!] $F does not pass clang-format format check." >&2
......@@ -76,7 +81,24 @@ clang-format:
if [ $RETURN -ne 0 ]; then exit 1; fi
'
dhall-format:
stage: source
tags:
- kvm
- nix
script:
- |
nix-shell --pure -p '(import ./.).fd' '(import ./.).haskellPackages.dhall' --run bash <<< '
RETURN=0
for F in $(fd -E hsnrm/hbandit -e dhall); do
dhall format < $F | cmp -s $F -
if [ $? -ne 0 ]; then
echo "[!] $F does not pass clang-format format check." >&2
RETURN=1
fi
done
if [ $RETURN -ne 0 ]; then exit 1; fi
'
nix-build-nrm.so:
stage: lib
......@@ -110,13 +132,13 @@ check-vendored-documentation:
- nix
script:
- |
nix-shell dev/default.nix -A hsnrm-hack --run bash <<< '
nix-shell dev/default.nix -A hack --run bash <<< '
echo "check: is vendored documentation up to date?"
cd hsnrm
cabal v2-haddock nrm.so --haddock-hyperlink-source --builddir=../doc_tmp
diff -r ../doc_tmp/build/x86_64-linux/ghc-8.6.5/hsnrm-1.0.0/x/nrm.so/doc/html/hsnrm/nrm.so ../doc/nrm.so/haddocks
rm -rf ../doc_tmp
'
cp -r doc/ doc2
./shake doc
./shake notebooks
diff -r doc doc2
'
nix-build-resources:
stage: codegen
......@@ -229,6 +251,7 @@ readthedocs:
when: on_success
only:
- master
- staging
tags:
- kvm
- nix
......
[submodule "hsnrm/hbandit"]
path = hsnrm/hbandit
url = ../hbandit.git
......@@ -14,7 +14,7 @@ libnrm-instrumented applications can also be installed using `nix-env -f. -iA s
### Development
from a local clone:
from a local clone, *with submodules initialized and updated*:
##### read internal haddock documentation:
......@@ -53,3 +53,21 @@ your latest local commit.
##### install NRM to a (Nix-enabled) remote system user environment:
`./deploy nrm user@host`
##### jupyter notebooks:
Tutorial Notebooks:
* Unexecuted: [`notebooks/raw`](notebooks/raw)
* Executed: [`notebooks/executed`](notebooks/executed)
Exploratory Notebooks:
* Unexecuted: [`notebooks/exploratory`](notebooks/exploratory)
* Running the notebooks in development mode:
```
./shake build
./shake client
./shake pyclient
nix-shell --run "jupyter-notebook <notebook file>"
```
......@@ -4,112 +4,132 @@ let types = ./dhall-to-cabal/types.dhall
let common = ./common.dhall
in \(ghcPath:Text) -> \(ghcNumericVersion:Text) -> prelude.defaults.Package
⫽ { name =
"hsnrm"
, version =
prelude.v "1.0.0"
, author =
"Valentin Reis"
, build-type =
Some types.BuildType.Simple
, cabal-version =
prelude.v "2.0"
, category =
"tools"
, description =
"The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
, executables =
[ { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Export.hs"
, build-depends =
[common.nobound "nrmlib", common.deps.base, common.deps.protolude , common.deps.enclosed-exceptions ]
, hs-source-dirs =
[ "bin" ]
}
⫽ common.copts
[ "-Wmissed-specialisations"
, "-Wall-missed-specialisations"
, "-fPIC"
, "-shared"
, "-no-hs-main"
, "-dynamic"
, "-lHSrts-ghc" ++ ghcNumericVersion
, "-L" ++ ghcPath ++ "/lib/ghc-" ++ ghcNumericVersion ++ "/rts/"
]
, name =
"nrm.so"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Hnrm.hs"
, build-depends =
[common.nobound "nrmlib"]
, hs-source-dirs =
[ "bin" ]
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrmstatic"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrm.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrm"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrmd.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrmd" ]
, name =
"fake-daemon"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Codegen.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Codegen" ]
, name =
"codegen"
}
]
, extra-source-files =
[ "ChangeLog.md" ]
, license =
types.License.BSD3
, license-files =
[] : List Text
, maintainer =
"fre@freux.fr"
, source-repos =
[ prelude.defaults.SourceRepo
⫽ { type =
Some types.RepoType.Git
, location =
Some "https://xgitlab.cels.anl.gov/vreis/hsnrm.git"
in λ ( ghcPath
: Text
)
→ λ(ghcNumericVersion : Text)
→ prelude.defaults.Package
⫽ { name =
"hsnrm"
, version =
prelude.v "1.0.0"
, author =
"Valentin Reis"
, build-type =
Some types.BuildType.Simple
, cabal-version =
prelude.v "2.0"
, category =
"tools"
, description =
"The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
, executables =
[ { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Export.hs"
, build-depends =
[ common.nobound "nrmlib"
, common.deps.base
, common.deps.protolude
, common.deps.enclosed-exceptions
]
, hs-source-dirs =
[ "bin" ]
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts
[ "-Wmissed-specialisations"
, "-Wall-missed-specialisations"
, "-fPIC"
, "-shared"
, "-no-hs-main"
, "-dynamic"
, "-lHSrts-ghc" ++ ghcNumericVersion
, "-L"
++ ghcPath
++ "/lib/ghc-"
++ ghcNumericVersion
++ "/rts/"
]
, name =
"nrm.so"
}
]
, synopsis =
"hsnrm"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Hnrm.hs"
, build-depends =
[ common.nobound "nrmlib" ]
, hs-source-dirs =
[ "bin", "src", "hbandit/src" ]
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrmstatic"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrm.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrm"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrmd.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrmd" ]
, name =
"fake-daemon"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Codegen.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Codegen" ]
, name =
"codegen"
}
]
, extra-source-files =
[ "ChangeLog.md" ]
, license =
types.License.BSD3
, license-files =
[] : List Text
, maintainer =
"fre@freux.fr"
, source-repos =
[ prelude.defaults.SourceRepo
⫽ { type =
Some types.RepoType.Git
, location =
Some "https://xgitlab.cels.anl.gov/vreis/hsnrm.git"
}
]
, synopsis =
"hsnrm"
}
......@@ -17,6 +17,7 @@ let defexts =
, types.Extension.OverloadedStrings True
, types.Extension.ViewPatterns True
, types.Extension.DeriveFunctor True
, types.Extension.DeriveTraversable True
, types.Extension.TypeFamilies True
, types.Extension.DeriveAnyClass True
, types.Extension.DeriveGeneric True
......@@ -110,6 +111,8 @@ let deps =
nobound "hxt-xpath"
, recursion-schemes =
nobound "recursion-schemes"
, binary =
nobound "binary"
, refined =
nobound "refined"
, generic-data =
......@@ -250,6 +253,7 @@ let modules =
, "NRM.Types.DownstreamCmd"
, "NRM.Types.DownstreamCmdID"
, "NRM.Types.State"
, "NRM.Types.MemBuffer"
, "NRM.Types.UpstreamClient"
, "NRM.Types.DownstreamClient"
, "NRM.Classes.Messaging"
......@@ -261,6 +265,8 @@ let modules =
, "NRM.Orphans.UUID"
, "NRM.Orphans.Dhall"
, "NRM.Orphans.NonEmpty"
, "NRM.Orphans.ZeroOne"
, "NRM.Orphans.Refined"
, "NRM.Types.Messaging.DownstreamEvent"
, "NRM.Types.Messaging.UpstreamPub"
, "NRM.Types.Messaging.UpstreamReq"
......@@ -289,9 +295,33 @@ let extramodules =
, "CPD.Values"
, "CPD.Utils"
, "CPD.Text"
, "Control.Monad.LPMonad.Internal"
, "Control.Monad.LPMonad.Supply.Class"
, "Control.Monad.LPMonad.Supply"
, "Control.Monad.LPMonad"
, "Data.LinearProgram"
, "Data.LinearProgram.Common"
, "Data.LinearProgram.GLPK"
, "Data.LinearProgram.GLPK.Common"
, "Data.LinearProgram.GLPK.Internal"
, "Data.LinearProgram.GLPK.IO.Internal"
, "Data.LinearProgram.GLPK.IO"
, "Data.LinearProgram.GLPK.Solver"
, "Data.LinearProgram.GLPK.Types"
, "Data.LinearProgram.LinExpr"
, "Data.LinearProgram.Spec"
, "Data.LinearProgram.Types"
, "Algebra.Classes"
]
let banditmodules = [ "Bandit.Class", "Bandit.Exp3", "Bandit.EpsGreedy", "Bandit.Util" ]
let banditmodules =
[ "HBandit.Class"
, "HBandit.Exp3"
, "HBandit.EpsGreedy"
, "HBandit.Util"
, "HBandit.Types"
, "HBandit.BwCR"
]
let allmodules = modules # extramodules # banditmodules
......@@ -312,6 +342,7 @@ let libdep =
, deps.generic-lens
, deps.data-msgpack
, deps.containers
, deps.binary
, deps.unordered-containers
, deps.mtl
, deps.aeson
......@@ -355,7 +386,8 @@ let libdep =
in { defexts =
defexts
,deps = deps
, deps =
deps
, libdep =
libdep
, banditmodules =
......
......@@ -4,142 +4,167 @@ let types = ./dhall-to-cabal/types.dhall
let common = ./common.dhall
in \(ghcPath:Text) -> \(ghcNumericVersion:Text) -> prelude.defaults.Package
⫽ { name =
"hsnrm"
, version =
prelude.v "1.0.0"
, author =
"Valentin Reis"
, build-type =
Some types.BuildType.Simple
, cabal-version =
prelude.v "2.0"
, category =
"tools"
, description =
"The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
, sub-libraries =
[ { library =
λ(config : types.Config)
→ prelude.defaults.Library
⫽ { build-depends =
common.libdep
, hs-source-dirs =
[ "nrm", "bin" ]
, exposed-modules =
common.allmodules
}
⫽ common.copts ([] : List Text)
, name =
"nrmlib"
}
, { library =
λ(config : types.Config)
→ prelude.defaults.Library
⫽ { build-depends =
common.libdep
, hs-source-dirs =
[ "nrm" ]
, exposed-modules =
common.banditmodules
}
⫽ common.copts ([] : List Text)
, name =
"banditlib"
}
]
, executables =
[ { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Export.hs"
, build-depends =
common.libdep
, hs-source-dirs =
[ "bin", "nrm" ]
, other-modules =
common.allmodules
}
⫽ common.copts
[ "-fPIC"
, "-shared"
, "-no-hs-main"
, "-dynamic"
]
, name =
"nrm.so"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Hnrm.hs"
, build-depends =
common.libdep
, hs-source-dirs =
[ "bin", "nrm" ]
, other-modules =
common.allmodules
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrmstatic"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrm.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrm"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrmd.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrmd" ]
, name =
"nrmddep"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Codegen.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Codegen" ]
, name =
"codegen"
}
]
, extra-source-files =
[ "ChangeLog.md" ]
, license =
types.License.BSD3
, license-files =
[] : List Text
, maintainer =
"fre@freux.fr"
, source-repos =
[ prelude.defaults.SourceRepo
⫽ { type =
Some types.RepoType.Git
, location =
Some "https://xgitlab.cels.anl.gov/vreis/hsnrm.git"
in λ ( ghcPath
: Text
)
→ λ(ghcNumericVersion : Text)
→ prelude.defaults.Package
⫽ { name =
"hsnrm"
, version =
prelude.v "1.0.0"
, author =
"Valentin Reis"
, build-type =
Some types.BuildType.Simple
, cabal-version =
prelude.v "2.0"
, category =
"tools"
, description =
"The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
, sub-libraries =
[ { library =
λ(config : types.Config)
→ prelude.defaults.Library
⫽ { build-depends =
common.libdep
, hs-source-dirs =
[ "nrm", "bin", "hbandit/src" ]
, exposed-modules =
common.allmodules
}
⫽ common.copts ([] : List Text)
, name =
"nrmlib"
}
]
, synopsis =
"hsnrm"
}
, { library =
λ(config : types.Config)
→ prelude.defaults.Library
⫽ { build-depends =
common.libdep
, hs-source-dirs =
[ "nrm" ]
, exposed-modules =
common.banditmodules
}
⫽ common.copts ([] : List Text)
, name =
"banditlib"
}
]
, executables =
[ { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Export.hs"
, build-depends =
common.libdep
, hs-source-dirs =
[ "bin", "nrm", "hbandit/src" ]