Commit 5db2b9db authored by Valentin Reis's avatar Valentin Reis
Browse files

[refactor] flipping configuration file generation strategy.

Flips the code generation around for configuration and manifest to be
defined in dhall files. GHC then evaluates a dhall type expression at compile
time. This allows for manual evolution of the Dhall code, which means easier
documentation, code factoring, code presentation, and reduction in amount
of generated files, so less duplication. Examples are also pulled out of the
Haskell source and refactored to be more presentable. Cabal file generation
is also removed.
parent f088a115
Pipeline #11743 passed with stages
in 2 minutes and 4 seconds
......@@ -8,7 +8,7 @@ let baseJob =
let mkJob =
λ(stage : Text)
→ λ(target : Text)
→ baseJob stage ("nix-shell -p gnumake --run 'make " ++ target ++"'")
→ baseJob stage ("nix-shell -p gnumake --run 'make " ++ target ++ "'")
let mkNix =
λ(stage : Text)
......@@ -23,66 +23,36 @@ let mkT = mkJob "test"
let mkNixB = mkNix "build"
in { stages =
[ "source", "build", "test", "deploy" ]
, nix/hsnrm-bin =
mkNixB "haskellPackages.hsnrm-bin"
, nix/hsnrm =
mkNixB "haskellPackages.hsnrm"
, nix/pynrm =
mkNixB "pythonPackages.pynrm"
, nix/libnrm =
mkNixB "libnrm"
, nix/stream =
mkNixB "stream"
, notebooks =
mkT "notebooks"
, tests/kvm =
mkT "tests-kvm"
, tests/apps =
mkT "app-tests"
, tests/rapl =
mkT "tests-rapl" // {tags = ["chimera"]}
, tests/perf =
mkT "tests-perf" // {tags = ["chimera"]}
, libnrm/autotools =
mkB "libnrm/autotools"
, hsnrm/all =
mkB "hsnrm/all"
, hsnrm/hsnrm-bin/hsnrm-bin_cabal =
mkS "hsnrm/hsnrm/hsnrm.cabal"
, hsnrm/hsnrm/hsnrm_cabal =
mkS "hsnrm/hsnrm-bin/hsnrm-bin.cabal"
, hsnrm/resources =
mkS "hsnrm/resources"
, shellcheck =
mkS "shellcheck"
, nixfmt =
mkS "nixfmt"
, dhall-format =
mkS "dhall-format"
, libnrm/clang-format =
mkS "libnrm/clang-format"
, pynrm/black =
mkS "pynrm/black"
, hsnrm/ormolu =
mkS "hsnrm/ormolu"
, hsnrm/hlint =
mkS "hsnrm/hlint"
, hsnrm/shellcheck =
mkS "hsnrm/shellcheck"
, hsnrm-/dhall-format =
mkS "hsnrm/dhall-format"
in { stages = [ "source", "build", "test", "deploy" ]
, nix/hsnrm-bin = mkNixB "haskellPackages.hsnrm-bin"
, nix/hsnrm = mkNixB "haskellPackages.hsnrm"
, nix/pynrm = mkNixB "pythonPackages.pynrm"
, nix/libnrm = mkNixB "libnrm"
, nix/stream = mkNixB "stream"
, notebooks = mkT "notebooks"
, tests/kvm = mkT "tests-kvm"
, tests/apps = mkT "app-tests"
, tests/rapl = mkT "tests-rapl" ⫽ { tags = [ "chimera" ] }
, tests/perf = mkT "tests-perf" ⫽ { tags = [ "chimera" ] }
, libnrm/autotools = mkB "libnrm/autotools"
, hsnrm/all = mkB "hsnrm/all"
, resources = mkS "resources"
, shellcheck = mkS "shellcheck"
, nixfmt = mkS "nixfmt"
, dhall-format = mkS "dhall-format"
, libnrm/clang-format = mkS "libnrm/clang-format"
, pynrm/black = mkS "pynrm/black"
, hsnrm/ormolu = mkS "hsnrm/ormolu"
, hsnrm/hlint = mkS "hsnrm/hlint"
, hsnrm/shellcheck = mkS "hsnrm/shellcheck"
, hsnrm-/dhall-format = mkS "hsnrm/dhall-format"
, readthedocs =
{ stage =
"deploy"
, tags =
tags
, only =
[ "master" ]
{ stage = "deploy"
, tags = tags
, only = [ "master" ]
, script =
[ "echo \"token=\$RTD_TOKEN\""
, "nix run nixpkgs.curl -c curl --fail -X POST -d \"token=\$RTD_TOKEN\" readthedocs.org/api/v2/webhook/hnrm/104604/"
]
[ "echo \"token=\$RTD_TOKEN\""
, "nix run nixpkgs.curl -c curl --fail -X POST -d \"token=\$RTD_TOKEN\" readthedocs.org/api/v2/webhook/hnrm/104604/"
]
}
}
......@@ -23,30 +23,12 @@ hsnrm/hlint:
tags:
- nix
- kvm
hsnrm/hsnrm-bin/hsnrm-bin_cabal:
script: "nix-shell -p gnumake --run 'make hsnrm/hsnrm/hsnrm.cabal'"
stage: source
tags:
- nix
- kvm
hsnrm/hsnrm/hsnrm_cabal:
script: "nix-shell -p gnumake --run 'make hsnrm/hsnrm-bin/hsnrm-bin.cabal'"
stage: source
tags:
- nix
- kvm
hsnrm/ormolu:
script: "nix-shell -p gnumake --run 'make hsnrm/ormolu'"
stage: source
tags:
- nix
- kvm
hsnrm/resources:
script: "nix-shell -p gnumake --run 'make hsnrm/resources'"
stage: source
tags:
- nix
- kvm
hsnrm/shellcheck:
script: "nix-shell -p gnumake --run 'make hsnrm/shellcheck'"
stage: source
......@@ -123,6 +105,12 @@ readthedocs:
tags:
- nix
- kvm
resources:
script: "nix-shell -p gnumake --run 'make resources'"
stage: source
tags:
- nix
- kvm
shellcheck:
script: "nix-shell -p gnumake --run 'make shellcheck'"
stage: source
......
......@@ -107,13 +107,24 @@ dhall-format:
resource-propagation: pynrm/nrm/schemas/downstreamEvent.json\
libnrm/src/nrm_messaging.h
pynrm/nrm/schemas/downstreamEvent.json: hsnrm/resources
cp hsnrm/resources/downstreamEvent.json $@
libnrm/src/nrm_messaging.h: hsnrm/resources
cp hsnrm/resources/nrm_messaging.h $@
pynrm/nrm/schemas/downstreamEvent.json: resources
cp resources/schemas/downstream.json $@
libnrm/src/nrm_messaging.h: resources
cp resources/nrm_messaging.h $@
resources:
@nix-shell --pure -E '
let pkgs = import <nixpkgs> {};
in with pkgs; haskellPackages.shellFor {
packages = p: [ haskellPackages.hsnrm haskellPackages.hsnrm-extra ];
buildInputs = [ cabal-install ];
}
' --run bash <<< '
cd hsnrm; cabal v2-run hsnrm-extra/Codegen.hs ../resources
'
############################# SECTION: libnrm pseudo-recursive targets (actual directory uses autotools)
############################# SECTION: libnrm pseudo-recursive targets (directory uses autotools)
libnrm/all: libnrm/autotools
......@@ -157,7 +168,7 @@ libnrm/clang-format:
.PHONY: ci
ci:
@nix-shell --pure -p yq jq --run bash <<< '
@nix-shell -p yq jq --run bash <<< '
for jobname in $$(yq -r "keys| .[]" .gitlab-ci.yml); do
if [ "$$jobname" != "stages" ]; then
gitlab-runner exec shell "$$jobname"
......
......@@ -15,6 +15,37 @@ git clone https://xgitlab.cels.anl.gov/argo/hnrm.git
- local nix build: `nix-build -A nrm`
- local nix profile install: `nix-env -f. -iA nrm`
## Usage
### binaries
```
#daemon
nrmd --help
#client
nrm --help
```
### `nrmd` configuration / `nrm` manifests
`nrmd` configurations and `nrm` manifests can use any of YAML, JSON or Dhall.
JSON schemas (raw) and Dhall types (documented with comments, for now)
are available.
when using JSON/YAML,
- JSON schemas: `resources/schemas/{nrmd,manifest}.json`
- default values: `resources/defauts/{nrmd,manifest}.json`
when using Dhall,
- types: `hsnrm/hsnrm/dhall/types/{nrmd,manifest}.dhall`
- default values: `hsnrm/hsnrm/dhall/defaults/{nrmd,manifest}.dhall`
Example `nrmd` configurations and `nrm` manifests are available in the
`examples` folder in JSON,YAML and Dhall format.
### APIs
JSON schema files for upstream and downstream APIs are available.
- JSON/YAML schemas: `resources/schemas/{upstream-*,downstream}.json`
## Development
- building: `make`
......
let types = ../../hsnrm/hsnrm/dhall/types/manifest.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/manifest.dhall
in default
⫽ { name = "libnrm-wrapped"
, app =
default.app
⫽ { instrumentation = Some { ratelimit.hertz = 1000000.0 } }
}
: types.Manifest
{
"name": "libnrm-wrapped",
"app": {
"instrumentation": {
"ratelimit": {
"hertz": 1000000
}
}
}
}
name: libnrm-wrapped
app:
instrumentation:
ratelimit:
hertz: 1000000
let types = ../../hsnrm/hsnrm/dhall/types/manifest.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/manifest.dhall
in default
⫽ { name = "linuxperf-wrapped"
, app =
default.app
⫽ { perfwrapper = Some { perfFreq.hertz = 1.0, perfLimit = +100000 }
}
}
: types.Manifest
{
"name": "linuxperf-wrapped",
"app": {
"perfwrapper": {
"perfFreq": {
"hertz": 1
},
"perfLimit": 100000
}
}
}
name: linuxperf-wrapped
app:
perfwrapper:
perfFreq:
hertz: 1
perfLimit: 100000
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
let action1 = { microwatts = 1.0e8 }
let action2 = { microwatts = 2.0e8 }
in default
⫽ { raplCfg = Some
{ raplPath = "/sys/devices/virtual/powercap/intel-rapl"
, raplActions = [ action1, action2 ]
, referencePower.microwatts = 2.5e8
}
, controlCfg =
types.ControlCfg.ControlCfg
{ minimumControlInterval.microseconds = 100000.0
, staticPower.microwatts = 2.0e8
, learnCfg = types.LearnCfg.Contextual { horizon = +4000 }
, speedThreshold = 1.1
, referenceMeasurementRoundInterval = +6
, hint = types.Hint.Full
}
, activeSensorFrequency.hertz = 1.0
}
: types.Cfg
{
"activeSensorFrequency": {
"hertz": 1
},
"controlCfg": {
"hint": "Full",
"learnCfg": {
"horizon": 4000
},
"minimumControlInterval": {
"microseconds": 100000
},
"referenceMeasurementRoundInterval": 6,
"speedThreshold": 1.1,
"staticPower": {
"microwatts": 200000000
}
},
"raplCfg": {
"raplActions": [
{
"microwatts": 100000000
},
{
"microwatts": 200000000
}
],
"raplPath": "/sys/devices/virtual/powercap/intel-rapl",
"referencePower": {
"microwatts": 250000000
}
}
}
activeSensorFrequency:
hertz: 1
controlCfg:
hint: Full
learnCfg:
horizon: 4000
minimumControlInterval:
microseconds: 100000
referenceMeasurementRoundInterval: 6
speedThreshold: 1.1
staticPower:
microwatts: 200000000
raplCfg:
raplActions:
- microwatts: 100000000
- microwatts: 200000000
raplPath: /sys/devices/virtual/powercap/intel-rapl
referencePower:
microwatts: 250000000
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
in default
⫽ { extraStaticActuators =
[ { staticActuatorKey = "example extra actuator ID"
, staticActuatorValue =
{ actuatorBinary = "bash"
, actuatorArguments =
[ "-c"
, "echo \$@ >> /tmp/test-nrm-example-extra-actuator"
, "-o"
]
, actions = [ 1.0, 2.0 ]
, referenceAction = 1.0
}
}
]
}
: types.Cfg
{
"extraStaticActuators": [
{
"staticActuatorKey": "example extra actuator ID",
"staticActuatorValue": {
"actions": [
1,
2
],
"actuatorArguments": [
"-c",
"echo $@ >> /tmp/test-nrm-example-extra-actuator",
"-o"
],
"actuatorBinary": "bash",
"referenceAction": 1
}
}
]
}
extraStaticActuators:
- staticActuatorKey: "example extra actuator ID"
staticActuatorValue:
actions:
- 1
- 2
actuatorArguments:
- "-c"
- "echo $@ >> /tmp/test-nrm-example-extra-actuator"
- "-o"
actuatorBinary: bash
referenceAction: 1
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
in default
⫽ { extraStaticPassiveSensors =
[ { passiveSensorKey = "example extra static passive power sensor"
, passiveSensorValue =
{ sensorBinary = "echo"
, sensorArguments = [ "30" ]
, range = { lower = 1.0, upper = 40.0 }
, tags = [ types.Tag.TagPower ]
, sensorBehavior = types.SensorBehavior.IntervalBased
}
}
]
}
: types.Cfg
{
"extraStaticPassiveSensors": [
{
"passiveSensorKey": "example extra static passive power sensor",
"passiveSensorValue": {
"range": {
"lower": 1,
"upper": 40
},
"sensorArguments": [
"30"
],
"sensorBehavior": "IntervalBased",
"sensorBinary": "echo",
"tags": [
"TagPower"
]
}
}
]
}
extraStaticPassiveSensors:
- passiveSensorKey: "example extra static passive power sensor"
passiveSensorValue:
range:
lower: 1
upper: 40
sensorArguments:
- "30"
sensorBehavior: IntervalBased
sensorBinary: echo
tags:
- TagPower
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
in default
⫽ { extraStaticPassiveSensors =
[ { passiveSensorKey =
"Sensor that gets package power limits for package 0 through variorum"
, passiveSensorValue =
{ sensorBinary = "bash"
, sensorArguments =
[ "-c"
, "variorum-print-power-limits-example | awk '{ if (\$1 == \"_PACKAGE_POWER_LIMITS\" && \$2 == \"0x610\" && \$4 == 0 ) { print \$6 } }'"
]
, range = { lower = 1.0, upper = 40.0 }
, tags = [] : List types.Tag
, sensorBehavior = types.SensorBehavior.IntervalBased
}
}
, { passiveSensorKey =
"Sensor that gets package power limits for package 1 through variorum"
, passiveSensorValue =
{ sensorBinary = "bash"
, sensorArguments =
[ "-c"
, "variorum-print-power-limits-example | awk '{ if (\$1 == \"_PACKAGE_POWER_LIMITS\" && \$2 == \"0x610\" && \$4 == 1 ) { print \$6 } }'"
]
, range = { lower = 1.0, upper = 40.0 }
, tags = [] : List types.Tag
, sensorBehavior = types.SensorBehavior.IntervalBased
}
}
]
}
: types.Cfg
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment