Commit 27b4a010 authored by Valentin Reis's avatar Valentin Reis
Browse files

Merge branch 'config' into 'master'

Configuration format restructuring & documentation / removing some codegen steps

See merge request !63
parents f088a115 5db2b9db
Pipeline #11745 passed with stages
in 1 minute and 28 seconds
......@@ -8,7 +8,7 @@ let baseJob =
let mkJob =
λ(stage : Text)
→ λ(target : Text)
→ baseJob stage ("nix-shell -p gnumake --run 'make " ++ target ++"'")
→ baseJob stage ("nix-shell -p gnumake --run 'make " ++ target ++ "'")
let mkNix =
λ(stage : Text)
......@@ -23,66 +23,36 @@ let mkT = mkJob "test"
let mkNixB = mkNix "build"
in { stages =
[ "source", "build", "test", "deploy" ]
, nix/hsnrm-bin =
mkNixB "haskellPackages.hsnrm-bin"
, nix/hsnrm =
mkNixB "haskellPackages.hsnrm"
, nix/pynrm =
mkNixB "pythonPackages.pynrm"
, nix/libnrm =
mkNixB "libnrm"
, nix/stream =
mkNixB "stream"
, notebooks =
mkT "notebooks"
, tests/kvm =
mkT "tests-kvm"
, tests/apps =
mkT "app-tests"
, tests/rapl =
mkT "tests-rapl" // {tags = ["chimera"]}
, tests/perf =
mkT "tests-perf" // {tags = ["chimera"]}
, libnrm/autotools =
mkB "libnrm/autotools"
, hsnrm/all =
mkB "hsnrm/all"
, hsnrm/hsnrm-bin/hsnrm-bin_cabal =
mkS "hsnrm/hsnrm/hsnrm.cabal"
, hsnrm/hsnrm/hsnrm_cabal =
mkS "hsnrm/hsnrm-bin/hsnrm-bin.cabal"
, hsnrm/resources =
mkS "hsnrm/resources"
, shellcheck =
mkS "shellcheck"
, nixfmt =
mkS "nixfmt"
, dhall-format =
mkS "dhall-format"
, libnrm/clang-format =
mkS "libnrm/clang-format"
, pynrm/black =
mkS "pynrm/black"
, hsnrm/ormolu =
mkS "hsnrm/ormolu"
, hsnrm/hlint =
mkS "hsnrm/hlint"
, hsnrm/shellcheck =
mkS "hsnrm/shellcheck"
, hsnrm-/dhall-format =
mkS "hsnrm/dhall-format"
in { stages = [ "source", "build", "test", "deploy" ]
, nix/hsnrm-bin = mkNixB "haskellPackages.hsnrm-bin"
, nix/hsnrm = mkNixB "haskellPackages.hsnrm"
, nix/pynrm = mkNixB "pythonPackages.pynrm"
, nix/libnrm = mkNixB "libnrm"
, nix/stream = mkNixB "stream"
, notebooks = mkT "notebooks"
, tests/kvm = mkT "tests-kvm"
, tests/apps = mkT "app-tests"
, tests/rapl = mkT "tests-rapl" ⫽ { tags = [ "chimera" ] }
, tests/perf = mkT "tests-perf" ⫽ { tags = [ "chimera" ] }
, libnrm/autotools = mkB "libnrm/autotools"
, hsnrm/all = mkB "hsnrm/all"
, resources = mkS "resources"
, shellcheck = mkS "shellcheck"
, nixfmt = mkS "nixfmt"
, dhall-format = mkS "dhall-format"
, libnrm/clang-format = mkS "libnrm/clang-format"
, pynrm/black = mkS "pynrm/black"
, hsnrm/ormolu = mkS "hsnrm/ormolu"
, hsnrm/hlint = mkS "hsnrm/hlint"
, hsnrm/shellcheck = mkS "hsnrm/shellcheck"
, hsnrm-/dhall-format = mkS "hsnrm/dhall-format"
, readthedocs =
{ stage =
"deploy"
, tags =
tags
, only =
[ "master" ]
{ stage = "deploy"
, tags = tags
, only = [ "master" ]
, script =
[ "echo \"token=\$RTD_TOKEN\""
, "nix run nixpkgs.curl -c curl --fail -X POST -d \"token=\$RTD_TOKEN\" readthedocs.org/api/v2/webhook/hnrm/104604/"
]
[ "echo \"token=\$RTD_TOKEN\""
, "nix run nixpkgs.curl -c curl --fail -X POST -d \"token=\$RTD_TOKEN\" readthedocs.org/api/v2/webhook/hnrm/104604/"
]
}
}
......@@ -23,30 +23,12 @@ hsnrm/hlint:
tags:
- nix
- kvm
hsnrm/hsnrm-bin/hsnrm-bin_cabal:
script: "nix-shell -p gnumake --run 'make hsnrm/hsnrm/hsnrm.cabal'"
stage: source
tags:
- nix
- kvm
hsnrm/hsnrm/hsnrm_cabal:
script: "nix-shell -p gnumake --run 'make hsnrm/hsnrm-bin/hsnrm-bin.cabal'"
stage: source
tags:
- nix
- kvm
hsnrm/ormolu:
script: "nix-shell -p gnumake --run 'make hsnrm/ormolu'"
stage: source
tags:
- nix
- kvm
hsnrm/resources:
script: "nix-shell -p gnumake --run 'make hsnrm/resources'"
stage: source
tags:
- nix
- kvm
hsnrm/shellcheck:
script: "nix-shell -p gnumake --run 'make hsnrm/shellcheck'"
stage: source
......@@ -123,6 +105,12 @@ readthedocs:
tags:
- nix
- kvm
resources:
script: "nix-shell -p gnumake --run 'make resources'"
stage: source
tags:
- nix
- kvm
shellcheck:
script: "nix-shell -p gnumake --run 'make shellcheck'"
stage: source
......
......@@ -107,13 +107,24 @@ dhall-format:
resource-propagation: pynrm/nrm/schemas/downstreamEvent.json\
libnrm/src/nrm_messaging.h
pynrm/nrm/schemas/downstreamEvent.json: hsnrm/resources
cp hsnrm/resources/downstreamEvent.json $@
libnrm/src/nrm_messaging.h: hsnrm/resources
cp hsnrm/resources/nrm_messaging.h $@
pynrm/nrm/schemas/downstreamEvent.json: resources
cp resources/schemas/downstream.json $@
libnrm/src/nrm_messaging.h: resources
cp resources/nrm_messaging.h $@
resources:
@nix-shell --pure -E '
let pkgs = import <nixpkgs> {};
in with pkgs; haskellPackages.shellFor {
packages = p: [ haskellPackages.hsnrm haskellPackages.hsnrm-extra ];
buildInputs = [ cabal-install ];
}
' --run bash <<< '
cd hsnrm; cabal v2-run hsnrm-extra/Codegen.hs ../resources
'
############################# SECTION: libnrm pseudo-recursive targets (actual directory uses autotools)
############################# SECTION: libnrm pseudo-recursive targets (directory uses autotools)
libnrm/all: libnrm/autotools
......@@ -157,7 +168,7 @@ libnrm/clang-format:
.PHONY: ci
ci:
@nix-shell --pure -p yq jq --run bash <<< '
@nix-shell -p yq jq --run bash <<< '
for jobname in $$(yq -r "keys| .[]" .gitlab-ci.yml); do
if [ "$$jobname" != "stages" ]; then
gitlab-runner exec shell "$$jobname"
......
......@@ -15,6 +15,37 @@ git clone https://xgitlab.cels.anl.gov/argo/hnrm.git
- local nix build: `nix-build -A nrm`
- local nix profile install: `nix-env -f. -iA nrm`
## Usage
### binaries
```
#daemon
nrmd --help
#client
nrm --help
```
### `nrmd` configuration / `nrm` manifests
`nrmd` configurations and `nrm` manifests can use any of YAML, JSON or Dhall.
JSON schemas (raw) and Dhall types (documented with comments, for now)
are available.
when using JSON/YAML,
- JSON schemas: `resources/schemas/{nrmd,manifest}.json`
- default values: `resources/defauts/{nrmd,manifest}.json`
when using Dhall,
- types: `hsnrm/hsnrm/dhall/types/{nrmd,manifest}.dhall`
- default values: `hsnrm/hsnrm/dhall/defaults/{nrmd,manifest}.dhall`
Example `nrmd` configurations and `nrm` manifests are available in the
`examples` folder in JSON,YAML and Dhall format.
### APIs
JSON schema files for upstream and downstream APIs are available.
- JSON/YAML schemas: `resources/schemas/{upstream-*,downstream}.json`
## Development
- building: `make`
......
let types = ../../hsnrm/hsnrm/dhall/types/manifest.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/manifest.dhall
in default
⫽ { name = "libnrm-wrapped"
, app =
default.app
⫽ { instrumentation = Some { ratelimit.hertz = 1000000.0 } }
}
: types.Manifest
{
"name": "libnrm-wrapped",
"app": {
"instrumentation": {
"ratelimit": {
"hertz": 1000000
}
}
}
}
name: libnrm-wrapped
app:
instrumentation:
ratelimit:
hertz: 1000000
let types = ../../hsnrm/hsnrm/dhall/types/manifest.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/manifest.dhall
in default
⫽ { name = "linuxperf-wrapped"
, app =
default.app
⫽ { perfwrapper = Some { perfFreq.hertz = 1.0, perfLimit = +100000 }
}
}
: types.Manifest
{
"name": "linuxperf-wrapped",
"app": {
"perfwrapper": {
"perfFreq": {
"hertz": 1
},
"perfLimit": 100000
}
}
}
name: linuxperf-wrapped
app:
perfwrapper:
perfFreq:
hertz: 1
perfLimit: 100000
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
let action1 = { microwatts = 1.0e8 }
let action2 = { microwatts = 2.0e8 }
in default
⫽ { raplCfg = Some
{ raplPath = "/sys/devices/virtual/powercap/intel-rapl"
, raplActions = [ action1, action2 ]
, referencePower.microwatts = 2.5e8
}
, controlCfg =
types.ControlCfg.ControlCfg
{ minimumControlInterval.microseconds = 100000.0
, staticPower.microwatts = 2.0e8
, learnCfg = types.LearnCfg.Contextual { horizon = +4000 }
, speedThreshold = 1.1
, referenceMeasurementRoundInterval = +6
, hint = types.Hint.Full
}
, activeSensorFrequency.hertz = 1.0
}
: types.Cfg
{
"activeSensorFrequency": {
"hertz": 1
},
"controlCfg": {
"hint": "Full",
"learnCfg": {
"horizon": 4000
},
"minimumControlInterval": {
"microseconds": 100000
},
"referenceMeasurementRoundInterval": 6,
"speedThreshold": 1.1,
"staticPower": {
"microwatts": 200000000
}
},
"raplCfg": {
"raplActions": [
{
"microwatts": 100000000
},
{
"microwatts": 200000000
}
],
"raplPath": "/sys/devices/virtual/powercap/intel-rapl",
"referencePower": {
"microwatts": 250000000
}
}
}
activeSensorFrequency:
hertz: 1
controlCfg:
hint: Full
learnCfg:
horizon: 4000
minimumControlInterval:
microseconds: 100000
referenceMeasurementRoundInterval: 6
speedThreshold: 1.1
staticPower:
microwatts: 200000000
raplCfg:
raplActions:
- microwatts: 100000000
- microwatts: 200000000
raplPath: /sys/devices/virtual/powercap/intel-rapl
referencePower:
microwatts: 250000000
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
in default
⫽ { extraStaticActuators =
[ { staticActuatorKey = "example extra actuator ID"
, staticActuatorValue =
{ actuatorBinary = "bash"
, actuatorArguments =
[ "-c"
, "echo \$@ >> /tmp/test-nrm-example-extra-actuator"
, "-o"
]
, actions = [ 1.0, 2.0 ]
, referenceAction = 1.0
}
}
]
}
: types.Cfg
{
"extraStaticActuators": [
{
"staticActuatorKey": "example extra actuator ID",
"staticActuatorValue": {
"actions": [
1,
2
],
"actuatorArguments": [
"-c",
"echo $@ >> /tmp/test-nrm-example-extra-actuator",
"-o"
],
"actuatorBinary": "bash",
"referenceAction": 1
}
}
]
}
extraStaticActuators:
- staticActuatorKey: "example extra actuator ID"
staticActuatorValue:
actions:
- 1
- 2
actuatorArguments:
- "-c"
- "echo $@ >> /tmp/test-nrm-example-extra-actuator"
- "-o"
actuatorBinary: bash
referenceAction: 1
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
in default
⫽ { extraStaticPassiveSensors =
[ { passiveSensorKey = "example extra static passive power sensor"
, passiveSensorValue =
{ sensorBinary = "echo"
, sensorArguments = [ "30" ]
, range = { lower = 1.0, upper = 40.0 }
, tags = [ types.Tag.TagPower ]
, sensorBehavior = types.SensorBehavior.IntervalBased
}
}
]
}
: types.Cfg
{
"extraStaticPassiveSensors": [
{
"passiveSensorKey": "example extra static passive power sensor",
"passiveSensorValue": {
"range": {
"lower": 1,
"upper": 40
},
"sensorArguments": [
"30"
],
"sensorBehavior": "IntervalBased",
"sensorBinary": "echo",
"tags": [
"TagPower"
]
}
}
]
}
extraStaticPassiveSensors:
- passiveSensorKey: "example extra static passive power sensor"
passiveSensorValue:
range:
lower: 1
upper: 40
sensorArguments:
- "30"
sensorBehavior: IntervalBased
sensorBinary: echo
tags:
- TagPower
let types = ../../hsnrm/hsnrm/dhall/types/nrmd.dhall
let default = ../../hsnrm/hsnrm/dhall/defaults/nrmd.dhall
in default
⫽ { extraStaticPassiveSensors =
[ { passiveSensorKey =
"Sensor that gets package power limits for package 0 through variorum"
, passiveSensorValue =
{ sensorBinary = "bash"
, sensorArguments =
[ "-c"
, "variorum-print-power-limits-example | awk '{ if (\$1 == \"_PACKAGE_POWER_LIMITS\" && \$2 == \"0x610\" && \$4 == 0 ) { print \$6 } }'"
]
, range = { lower = 1.0, upper = 40.0 }
, tags = [] : List types.Tag
, sensorBehavior = types.SensorBehavior.IntervalBased
}
}
, { passiveSensorKey =
"Sensor that gets package power limits for package 1 through variorum"
, passiveSensorValue =
{ sensorBinary = "bash"
, sensorArguments =
[ "-c"
, "variorum-print-power-limits-example | awk '{ if (\$1 == \"_PACKAGE_POWER_LIMITS\" && \$2 == \"0x610\" && \$4 == 1 ) { print \$6 } }'"
]
, range = { lower = 1.0, upper = 40.0 }
, tags = [] : List types.Tag
, sensorBehavior = types.SensorBehavior.IntervalBased
}
}
]
}
: types.Cfg
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment