Commit 752a0c49 authored by Valentin Reis's avatar Valentin Reis
Browse files

[fix] non-structural minor refactors, sensor range logic fix

also includes significant simplification of build chain
and CI rules.
parent 64e51632
Pipeline #10194 passed with stages
in 3 minutes and 12 seconds
......@@ -4,7 +4,7 @@ variables:
stages:
- source
- build
- integration
- tests
- vendoring
- deploy
......@@ -26,7 +26,7 @@ ormolu:
' --run bash <<< '
RETURN=0
for F in $(fd -E hsnrm/hbandit -E hsnrm/dhall-haskell -e hs); do
ormolu -o -XTypeApplications -m check $F
ormolu -o -XTypeApplications -o -XPatternSynonyms -m check $F
if [ $? -ne 0 ]; then
echo "[!] $F does not pass ormolu haskell format check." >&2
RETURN=1
......@@ -63,8 +63,8 @@ hlint:
script:
- |
nix-shell --pure -p '(import ./. {}).hlint' --run bash <<< '
hlint hsnrm/nrm/ --hint=./.hlint.yaml
hlint hsnrm/bin/ --hint=./.hlint.yaml
hlint hsnrm/nrm/ --hint=hsnrm/.hlint.yaml
hlint hsnrm/bin/ --hint=hsnrm/.hlint.yaml
'
black:
......@@ -118,14 +118,14 @@ dhall-format:
if [ $RETURN -ne 0 ]; then exit 1; fi
'
nix:nrm.so:
nix-build:nrm.so:
stage: build
tags:
- kvm
- nix
script: nix-build -A haskellPackages.nrmlib --no-build-output
nix:hs-nrmbin:
nix-build:hs-nrmbin:
stage: build
tags:
- kvm
......@@ -145,7 +145,7 @@ vendoring:copy:
- cp -r doc/ tmpdoc
- cp -r resources/ tmpresources
codegen:
make:codegen:
stage: build
artifacts:
paths:
......@@ -154,9 +154,9 @@ codegen:
- kvm
- nix
script:
- ./shake.sh codegen
- nix-shell -p 'gnumake' --run "make codegen"
doc:
make:doc:
stage: build
artifacts:
paths:
......@@ -165,9 +165,9 @@ doc:
- kvm
- nix
script:
- ./shake.sh doc
- nix-shell -p 'gnumake' --run "make doc"
pyclient:
make:pyclient:
stage: build
artifacts:
paths:
......@@ -176,9 +176,9 @@ pyclient:
- kvm
- nix
script:
- ./shake.sh pyclient
- nix-shell -p 'gnumake' --run "make pyclient"
build:
make:build:
stage: build
artifacts:
paths:
......@@ -187,10 +187,10 @@ build:
- kvm
- nix
script:
- ./shake.sh build
- nix-shell -p 'gnumake' --run "make build"
notebooks:
stage: integration
stage: tests
artifacts:
paths:
- doc
......@@ -198,7 +198,7 @@ notebooks:
- kvm
- nix
script:
- ./shake.sh notebooks
- nix-shell -p 'gnumake' --run "make notebooks"
vendoring:haddock:
stage: vendoring
......@@ -215,21 +215,21 @@ vendoring:resources:
stage: vendoring
dependencies:
- vendoring:copy
- codegen
- make:codegen
tags:
- kvm
- nix
script:
- diff -r resources tmpresources
nix:resources:
nix-build:resources:
stage: build
tags:
- kvm
- nix
script: nix-build -A resources --no-build-output
libnrm:
autotools:libnrm:
stage: build
tags:
- kvm
......@@ -249,84 +249,54 @@ libnrm:
make
'
nix:libnrm:
nix-build:libnrm:
stage: build
tags:
- kvm
- nix
script: nix-build -A libnrm --no-build-output
nix:pynrm:
nix-build:pynrm:
stage: build
tags:
- kvm
- nix
script: nix-build -A pynrm --no-build-output
nix:app-stream:
nix-build:app-stream:
stage: build
tags:
- kvm
- nix
script: nix-build -A stream --no-build-output
nix:app-stream:
nix-build:nrmFull:
stage: build
tags:
- kvm
- nix
script: nix-build -A stream --no-build-output
nix:nrmFull:
stage: build
tags:
- integration
script: nix-build -A nrm --no-build-output
dhrun:hello:
stage: integration
stage: tests
tags:
- kvm
- nix
script:
- |
nix-shell --pure -p '(import ./. {}).nrm' -p '(import ./. {}).dhrun' --run "dhrun -i" <<< '
let all = ./dev/dhrun/all-tests.dh
"../dev/dhrun/assets/"
"../resources/defaults/Cfg.dhall // { verbose=<Normal|Verbose|Debug>.Debug }"
"../resources/examples/"
in all.hello
'
script: nix-shell -p gnumake --run "make dhrun-hello"
dhrun:exitcode:
stage: integration
stage: tests
tags:
- kvm
- nix
script:
- |
nix-shell --pure -p '(import ./. {}).nrm' -p '(import ./. {}).dhrun' --run "dhrun -i" <<< '
let all = ./dev/dhrun/all-tests.dh
"../dev/dhrun/assets/"
"../resources/defaults/Cfg.dhall // { verbose=<Normal|Verbose|Debug>.Debug }"
"../resources/examples/"
in all.exitcode
'
script: nix-shell -p gnumake --run "make dhrun-exitcode"
dhrun:listen:
stage: integration
stage: tests
tags:
- kvm
- nix
script:
- |
nix-shell --pure -p '(import ./. {}).nrm' -p '(import ./. {}).dhrun' --run "dhrun -i" <<< '
let all = ./dev/dhrun/all-tests.dh
"../dev/dhrun/assets/"
"../resources/defaults/Cfg.dhall // { verbose=<Normal|Verbose|Debug>.Debug }"
"../resources/examples/"
in all.listen
'
script: nix-shell -p gnumake --run "make dhrun-listen"
readthedocs:
stage: deploy
......
This diff is collapsed.
# this file was tested using GNUMAKE >= 4.2.1.
# this is necessary for using multi-line strings as command arguments.
SHELL := $(shell which bash)
# this allows omitting newlines.
.ONESHELL:
.PHONY: ci
ci:
@nix-shell -p yq -p jq --run bash <<< '
for jobname in $$(yq -r "keys| .[]" .gitlab-ci.yml); do
if [ "$$jobname" != "stages" ]; then
gitlab-runner exec shell "$$jobname"
fi
done
'
ci-%:
@nix-shell -A hack --run bash <<< '
gitlab-runner exec shell "$*"
'
.PHONY: ghcid
ghcid: ghcid-nrmlib
ghcid-%: hsnrm/hsnrm.cabal hsnrm/.hlint.yaml
@nix-shell -A hack --pure --run bash <<< '
ghcid -C hsnrm --command "cabal v2-repl $* " --restart=holt.cabal -l
'
.PHONY: pre-commit
pre-commit: ormolu dhall-format shellcheck black codegen doc build pyclient notebooks
.PHONY: shellcheck
shellcheck:
@nix-shell --pure -p '(import ./. {}).fd' '(import ./. {}).shellcheck' --run bash <<< '
for F in $$(fd -E hsnrm/hbandit -E hsnrm/dhall-haskell -e sh); do
shellcheck -s bash $$F
done
'
.PHONY: dhall-format
dhall-format:
@nix-shell --pure -p '(import ./. {}).fd' '(import ./. {}).haskellPackages.dhall' --run bash <<< '
for F in $$(fd -E hsnrm/hbandit -E hsnrm/dhall-haskell -e dhall); do
dhall format --inplace $$F
done
'
.PHONY: ormolu
ormolu:
@nix-shell --pure -E '
let pkgs = (import ./. {});
in pkgs.mkShell {
buildInputs = [pkgs.fd pkgs.ormolu];
shellHook =
"export LOCALE_ARCHIVE=$${pkgs.glibcLocales}/lib/locale/locale-archive \n" +
"export LANG=en_US.UTF-8";
}
' --run bash <<< '
for F in $$(fd -E hsnrm/hbandit -E hsnrm/dhall-haskell -e hs); do
ormolu -o -XTypeApplications -o -XPatternSynonyms -m inplace $$F
done
'
.PHONY: black
black:
@nix-shell --pure -p '(import ./. {}).python37Packages.black' --run bash <<< '
black pynrm/bin/*
black pynrm/nrm/*.py
'
.PHONY: codegen
codegen:
@nix-shell --pure -A hack --run <<< bash '
cd hsnrm
cabal v2-run --builddir=../.build_codegen codegen ../resources/
'
.PHONY: doc
doc:
@nix-shell --pure -A hack --run <<< bash '
cd hsnrm
cabal v2-haddock nrm.so --haddock-internal --builddir=../.build
'
.PHONY: vendor
vendor:
@nix-shell --pure -A hack --run <<< bash '
cp $$CABALFILE hsnrm/hsnrm.cabal
cp $$CABALFILE_LIB dev/pkgs/hnrm/lib.cabal
cp $$CABALFILE_BIN dev/pkgs/hnrm/bin.cabal
cp $$NIXFILE_LIB/default.nix dev/pkgs/hnrm/lib.nix
cp $$NIXFILE_BIN/default.nix dev/pkgs/hnrm/bin.nix
chmod +rw hsnrm/hsnrm.cabal dev/pkgs/hnrm/bin.nix dev/pkgs/hnrm/lib.nix dev/pkgs/hnrm/bin.cabal dev/pkgs/hnrm/lib.cabal
sed -i 's/src = .*/inherit src;/' dev/pkgs/hnrm/lib.nix
sed -i 's/src = .*/inherit src;/' dev/pkgs/hnrm/bin.nix
'
.PHONY: notebooks
notebooks:
@nix-shell --pure -A hack --run <<< bash '
notebooks/batchnb.py notebooks/configuration.ipynb
jupyter nbconvert doc/notebooks/notebooks/configuration.ipynb --output-dir=doc/notebooks/notebooks
rm doc/notebooks/notebooks/configuration.ipynb
jupyter nbconvert notebooks/tutorial.ipynb --output-dir=doc/notebooks/notebooks
jupyter nbconvert notebooks/internal-control.ipynb --output-dir=doc/notebooks/notebooks
'
dhrun-%:
rm -f hsnrm/.ghc*
@nix-shell --pure -p '(import ./. {}).nrm' -p '(import ./. {}).dhrun' -p bash --run "dhrun -i" <<< '
let all = ./dev/dhrun/all-tests.dh
"../dev/dhrun/assets/"
"../resources/defaults/Cfg.dhall // { verbose=<Normal|Verbose|Debug>.Debug }"
"../resources/examples/"
in all.exitcode
'
.PHONY:client
client:
@nix-shell --pure -A hack --run bash <<< '
cd hsnrm
cabal v2-build nrm --builddir=../.build
'
.PHONY: build
build:
@nix-shell --pure -A hack --run runhaskell <<< '
{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE OverloadedStrings #-}
import Control.Monad
import Data.Text (dropEnd, lines, splitOn, strip)
import Development.Shake hiding (getEnv)
import Development.Shake.FilePath
import Options.Applicative as OA
import Protolude
import System.Directory
import System.Environment (getEnv, withArgs)
import System.FilePath.Glob
import qualified System.IO as SIO ( BufferMode (..), hSetBuffering, stdout,)
import System.Posix.Process
import System.Process.Typed
import qualified Prelude
main :: IO()
main = do
version <- toS . strip . toS <$$> readProcessStdout_ "ghc --numeric-version"
ghcPathRaw <- strip . toS <$$> readProcessStdout_ "which ghc"
let ghcPath = dropEnd 8 ghcPathRaw
runProcess_ $$ setWorkingDir "hsnrm" $$ shell "cp -f $$CABALFILE hsnrm.cabal"
runProcess_ . setWorkingDir "hsnrm" $$ proc "cabal" [ "v2-build", "nrm.so", "--ghc-option=-lHSrts_thr-ghc" <> version, "--ghc-option=-L" <> toS ghcPath <> "/lib/ghc-" <> version <> "/rts/", "--builddir=../.build", "--jobs=4" ]
'
.PHONY: pyclient
pyclient:
@nix-shell --pure -A hack --run runhaskell <<< '
{-# LANGUAGE NoImplicitPrelude #-}
{-# LANGUAGE OverloadedStrings #-}
import Control.Monad
import Data.Text (dropEnd, lines, splitOn, strip)
import Development.Shake hiding (getEnv)
import Development.Shake.FilePath
import Options.Applicative as OA
import Protolude
import System.Directory
import System.Environment (getEnv, withArgs)
import System.FilePath.Glob
import qualified System.IO as SIO ( BufferMode (..), hSetBuffering, stdout,)
import System.Posix.Process
import System.Process.Typed
import qualified Prelude
main :: IO()
main = do
version <- toS . strip . toS <$$> readProcessStdout_ "ghc --numeric-version"
ghcPathRaw <- strip . toS <$$> readProcessStdout_ "which ghc"
let ghcPath = dropEnd 8 ghcPathRaw
runProcess_ . setWorkingDir "hsnrm" $$ proc "cabal" [ "v2-build", "pynrm.so", "--ghc-option=-lHSrts_thr-ghc" <> version, "--ghc-option=-L" <> toS ghcPath <> "/lib/ghc-" <> version <> "/rts/", "--builddir=../.build", "--jobs=4" ]
'
......@@ -17,17 +17,16 @@ git clone --recurse-submodules https://xgitlab.cels.anl.gov/argo/hnrm.git
## Development/Local builds
- opening jupyter notebooks: `nix-shell -A expe`, which provisions
`jupyter-notebook`.
- running CI jobs locally: `./ci.sh <jobname>` or all jobs using `./ci`
- running CI jobs locally: `make ci-<jobname>` or all jobs using `make ci`
(requires gitlab-runner)
- entering a development shell : `nix-shell -A hack`
- use `./shake.sh build` to build the `nrm.so` shared library (also runs the
code generation step for vendored resources)
- use `./shake.sh client` to build the `nrm` client
- use `./shake.sh pyclient` to build the Python shared library (for use by the
- use `make build` to build the `nrm.so` shared library
- use `make codegen` to regenerate vendored `./resources`
- use `make client` to build the `nrm` client
- use `make pyclient` to build the Python shared library (for use by the
python module `nrm.tooling`)
- opening jupyter notebooks: `nix-shell -A expe`, which provisions
`jupyter-notebook`.
the appropriate Nix `shellHooks` are in place in the `hack` nix derivation for
you to use the development build using the `nrm` and `nrmd` aliases on the CLI.
#!/usr/bin/env nix-shell
#! nix-shell -p yq -p jq -i bash
if [ $# -eq 0 ]
then
for jobname in $(yq -r 'keys| .[]' .gitlab-ci.yml); do
if [ "$jobname" != "stages" ]; then
gitlab-runner exec shell "$jobname"
fi
done
else
gitlab-runner exec shell "$1"
fi
#!/usr/bin/env bash
# shellcheck disable=SC2086
# shellcheck disable=SC2048
rm -f .ghc.env*
if [ -z "$IN_NIX_SHELL" ]
then
nix-shell \
--run "runhaskell dev/deploy_g5k.hs $*" \
--option extra-substituters http://129.114.24.212/serve \
--option trusted-public-keys example-nix-cache-1:HSwzbJmGDidTrax3Lvx1vMSvto04VN2O5cjfXAG9uz0=
else
runhaskell dev/deploy_g5k.hs $*
fi
#!/usr/bin/env bash
attr=$1
target=$2
storepath=$(nix-build -A "$attr")
echo "$storepath"
nix-copy-closure --to "$target" "$storepath"
ssh "$target" '$HOME/.nix-profile/bin/nix-env -i "$storepath"'
......@@ -37,10 +37,6 @@ in λ ( ghcPath
]
, hs-source-dirs =
[ "bin" ]
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts
[ "-Wmissed-specialisations"
......@@ -68,10 +64,6 @@ in λ ( ghcPath
[ common.nobound "nrmlib" ]
, hs-source-dirs =
[ "bin", "src", "hbandit/src" ]
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
......@@ -89,18 +81,6 @@ in λ ( ghcPath
, name =
"nrm"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"bin/Hnrmd.hs"
, build-depends =
[ common.nobound "nrmlib" ]
}
⫽ common.copts [ "-main-is", "Hnrmd" ]
, name =
"fake-daemon"
}
, { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
......
......@@ -8,6 +8,7 @@ let defexts =
, types.Extension.DefaultSignatures True
, types.Extension.ExistentialQuantification True
, types.Extension.RecordWildCards True
, types.Extension.RankNTypes True
, types.Extension.TypeSynonymInstances True
, types.Extension.StandaloneDeriving True
, types.Extension.FlexibleInstances True
......@@ -16,12 +17,14 @@ let defexts =
, types.Extension.ImplicitPrelude False
, types.Extension.OverloadedStrings True
, types.Extension.ViewPatterns True
, types.Extension.PatternSynonyms True
, types.Extension.DeriveFunctor True
, types.Extension.DeriveTraversable True
, types.Extension.TypeFamilies True
, types.Extension.DeriveAnyClass True
, types.Extension.DeriveGeneric True
, types.Extension.DeriveDataTypeable True
, types.Extension.OverloadedLabels True
, types.Extension.DeriveFoldable True
, types.Extension.DerivingStrategies True
, types.Extension.TypeApplications True
......@@ -297,23 +300,6 @@ let extramodules =
, "CPD.Values"
, "CPD.Utils"
, "CPD.Text"
, "Control.Monad.LPMonad.Internal"
, "Control.Monad.LPMonad.Supply.Class"
, "Control.Monad.LPMonad.Supply"
, "Control.Monad.LPMonad"
, "Data.LinearProgram"
, "Data.LinearProgram.Common"
, "Data.LinearProgram.GLPK"
, "Data.LinearProgram.GLPK.Common"
, "Data.LinearProgram.GLPK.Internal"
, "Data.LinearProgram.GLPK.IO.Internal"
, "Data.LinearProgram.GLPK.IO"
, "Data.LinearProgram.GLPK.Solver"
, "Data.LinearProgram.GLPK.Types"
, "Data.LinearProgram.LinExpr"
, "Data.LinearProgram.Spec"
, "Data.LinearProgram.Types"
, "Algebra.Classes"
]
let banditmodules =
......
......@@ -65,10 +65,6 @@ in λ ( ghcPath
[ "bin", "nrm", "hbandit/src" ]
, other-modules =
common.allmodules
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts
[ "-fPIC", "-shared", "-no-hs-main", "-dynamic" ]
......@@ -86,10 +82,6 @@ in λ ( ghcPath
[ "bin", "nrm", "hbandit/src" ]
, other-modules =
common.allmodules
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts
[ "-fPIC", "-shared", "-no-hs-main", "-dynamic" ]
......
......@@ -33,10 +33,6 @@ in λ ( ghcPath
[ "nrm", "hbandit/src" ]
, exposed-modules =
common.allmodules
, c-sources =
[ "glpk/glpk.c" ]
, extra-libraries =
[ "glpk" ]
}
⫽ common.copts ([] : List Text)
)
......
let prelude = ./dhall-to-cabal/prelude.dhall
let types = ./dhall-to-cabal/types.dhall
let common = ./common.dhall
in λ ( ghcPath
: Text
)
→ λ(ghcNumericVersion : Text)
→ prelude.defaults.Package
⫽ { name =
"hsnrm"
, version =
prelude.v "1.0.0"
, author =
"Valentin Reis"
, build-type =
Some types.BuildType.Simple
, cabal-version =
prelude.v "2.0"
, category =
"tools"
, description =
"The Node Resource Manager(NRM) is a linux daemon that enables dynamic resource optimization for improving the power/performance tradeoff of HPC applications."
, executables =
[ { executable =
λ(config : types.Config)
→ prelude.defaults.Executable
⫽ { main-is =
"Hnrm.hs"
, build-depends =
common.libdep
, hs-source-dirs =
[ "bin", "nrm", "hbandit/src" ]
, other-modules =
common.allmodules
}
⫽ common.copts [ "-main-is", "Hnrm" ]
, name =
"nrm"
}
]
, extra-source-files =
[ "ChangeLog.md" ]
, license =
types.License.BSD3