Commit ec10788d authored by Valentin Reis's avatar Valentin Reis
Browse files

Merge branch 'doc-formats' into 'master'

python upstream layer/ notebook updates

See merge request !67
parents d1bcddce 5b550345
Pipeline #11785 passed with stages
in 3 minutes and 58 seconds
_build
.mypy*
.Rhistory
tmp
.build
......
......@@ -29,7 +29,6 @@ in { stages = [ "source", "build", "test", "deploy" ]
, nix/pynrm = mkNixB "pythonPackages.pynrm"
, nix/libnrm = mkNixB "libnrm"
, nix/stream = mkNixB "stream"
, notebooks = mkT "notebooks"
, tests/kvm = mkT "tests-kvm"
, tests/apps = mkT "app-tests"
, tests/rapl = mkT "tests-rapl" ⫽ { tags = [ "chimera" ] }
......
......@@ -89,12 +89,6 @@ nixfmt:
tags:
- nix
- kvm
notebooks:
script: "nix-shell -p gnumake --run 'make notebooks'"
stage: test
tags:
- nix
- kvm
pynrm/black:
script: "nix-shell -p gnumake --run 'make pynrm/black'"
stage: source
......
......@@ -38,16 +38,6 @@ pre-commit: hsnrm/pre-commit\
resource-propagation\
.gitlab-ci.yml
.PHONY: notebooks
notebooks:
@nix-shell --pure --run <<< bash '
notebooks/batchnb.py notebooks/configuration.ipynb
jupyter nbconvert doc/notebooks/notebooks/configuration.ipynb --output-dir=doc/notebooks/notebooks
rm doc/notebooks/notebooks/configuration.ipynb
jupyter nbconvert notebooks/tutorial.ipynb --output-dir=doc/notebooks/notebooks
jupyter nbconvert notebooks/internal-control.ipynb --output-dir=doc/notebooks/notebooks
'
app-tests:
@nix-shell --pure -p \
'with (import <nixpkgs> {}); stream.override { nrmSupport = true; }' \
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
{
"activeSensorFrequency": {
"passiveSensorFrequency": {
"hertz": 1
},
"controlCfg": {
......
activeSensorFrequency:
passiveSensorFrequency:
hertz: 1
controlCfg:
hint: Full
......
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
import nrm.sharedlib
import signal
import os
import yaml
import json
import subprocess
import shutil
from contextlib import contextmanager
import nrm.messaging
from multiprocessing import Process
from typing import NamedTuple, List, NewType
ActuatorID = NewType("ActuatorID", str)
SensorID = NewType("SensorID", str)
ActuatorValue = NewType("ActuatorValue", float)
class Actuator(NamedTuple):
actuatorID: ActuatorID
admissibleActions: List[ActuatorValue]
class Sensor(NamedTuple):
sensorID: SensorID
maxFrequency: float
lowerbound: float
upperbound: float
class Action(NamedTuple):
actuatorID: ActuatorID
actuatorValue: ActuatorValue
......@@ -33,19 +48,35 @@ def _exitCodeBool(*args, **kwargs):
return _doesntThrow(subprocess.check_call, *args, **kwargs)
class CPD(object):
def __init__(self, cpd):
class CPD:
def __init__(self, cpd: str):
self.cpd = cpd
def __str__(self):
return lib.showCpd(self.cpd)
def __iter__(self):
for key, value in json.loads(lib.jsonCpd(self.cpd)).items():
yield key, value
yield from json.loads(lib.jsonCpd(self.cpd)).items()
def actuators(self) -> List[Actuator]:
return [
Actuator(actuatorID=a[0], admissibleActions=a[1]["actions"])
for a in json.loads(lib.jsonCpd(self.cpd))["actuators"]
]
def sensors(self) -> List[Sensor]:
return [
Sensor(
sensorID=a[0],
maxFrequency=a[1]["maxFrequency"],
lowerbound=a[1]["range"]["i"][0],
upperbound=a[1]["range"]["i"][1],
)
for a in json.loads(lib.jsonCpd(self.cpd))["sensors"]
]
class NRMState(object):
class NRMState:
def __init__(self, cpd):
self.state = cpd
......@@ -53,146 +84,89 @@ class NRMState(object):
return lib.showState(self.state)
def __iter__(self):
for key, value in json.loads(lib.jsonState(self.state)).items():
yield key, value
class Remote(object):
def __init__(self, target):
self.target = target
def start_daemon(self, configuration):
""" start nrmd """
if self.check_daemon():
self.stop_daemon()
return subprocess.check_call(
[
"ssh",
"%s" % (self.target),
"/home/cc/.nix-profile/bin/daemonize /home/cc/.nix-profile/bin/nrmd "
+ configuration,
],
stderr=subprocess.STDOUT,
yield from json.loads(lib.jsonState(self.state)).items()
@contextmanager
def nrmd(configuration):
"""
The nrmd context manager is the proper way to initialize the NRM daemon
via this module.
SIGTERM will be sent to the process that performed the resource acquisition
if the daemon terminates illegally.
Example:
with nrmd({}) as d:
cpd = d.get_cpd()
print(cpd.actuators())
print(cpd.sensors())
print(d.upstream_recv())
print("done.")
"""
def daemon():
subprocess.run(["pkill", "-f", "nrmd"])
subprocess.run(["pkill", "nrmd"])
completed = subprocess.run(
[shutil.which("nrmd"), "-y", json.dumps(configuration),]
)
if completed.returncode != 0:
print("NRM daemon exited with exit code %d" % completed.returncode)
os.kill(os.getppid(), signal.SIGTERM)
def check_daemon(self):
""" checks if nrmd is alive """
return _exitCodeBool(["ssh", "%s" % self.target, "pgrep nrmd"])
def stop_daemon(self):
""" stops nrmd """
subprocess.check_call(["ssh", "%s" % self.target, "pkill nrmd"])
def run_workload(self, workload):
""" Runs a workload via NRM. The `nrmd` daemon must be running. """
pass
def get_state(self):
pass
def get_cpd(self):
pass
def workload_finished(self):
""" Checks NRM to see whether all tasks are finished. """
return True
def workload_recv(self):
""" Receive a message from NRM's upstream API. """
pass
def workload_send(self, message):
""" Send a message to NRM's upstream API. """
pass
def workload_exit_status(self):
""" Check the workload's exit status. """
pass
p = Process(target=daemon)
p.start()
try:
yield NRMD(p)
finally:
p.kill()
subprocess.run(["pkill", "-f", "nrmd"])
subprocess.run(["pkill", "nrmd"])
class Local(object):
def __init__(self):
class NRMD:
def __init__(self, daemon):
self.daemon = daemon
self.commonOpts = lib.defaultCommonOpts()
def start_daemon(
self, configuration, outfile="/tmp/nrmd_out", errfile="/tmp/nrmd_err"
):
""" start nrmd """
if self.check_daemon():
self.stop_daemon()
subprocess.Popen(
[
"daemonize",
"-o",
outfile,
"-e",
errfile,
shutil.which("nrmd"),
"-y",
json.dumps(configuration),
],
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
)
self.upstreampub = nrm.messaging.UpstreamPubClient(
lib.pubAddress(self.commonOpts)
)
print("connecting")
self.upstreampub.connect(wait=False)
print("connected to %s" % lib.pubAddress(self.commonOpts))
return
def check_daemon(self):
""" checks if nrmd is alive """
return _exitCodeBool(["pgrep", "-f", "nrmd"]) or _exitCodeBool(
["pgrep", "nrmd"]
def run(self, cmd: str, args: List[str], manifest, sliceID: str):
""" Upstream request: Run an application via NRM."""
lib.run(
self.commonOpts,
lib.mkSimpleRun(
cmd,
args,
list(dict(os.environ).items()),
json.dumps(manifest),
sliceID,
),
)
def stop_daemon(self):
""" stops nrmd """
if not (
_exitCodeBool(["pkill", "-f", "nrmd"]) or _exitCodeBool(["pkill", "nrmd"])
):
raise (Exception)
def run_workload(self, workloads):
""" Runs a workload via NRM. The `nrmd` daemon must be running. """
for w in workloads:
lib.run(
self.commonOpts,
lib.mkSimpleRun(
w["cmd"],
w["args"],
list(dict(os.environ).items()),
json.dumps(w["manifest"]),
w["sliceID"],
),
)
def actuate(self, actionList: List[Action]) -> None:
""" Upstream request: Run an available action """
actionList = [(str(a.actuatorID), float(a.actuatorValue)) for a in actionList]
if lib.action(self.commonOpts, actionList):
pass
else:
raise (Exception("couldn't actuate"))
def get_cpd(self):
""" Obtain the current Control Problem Description """
def get_cpd(self) -> str:
""" Upstream request: Obtain the current Control Problem Description """
return CPD(lib.cpd(self.commonOpts))
def get_state(self):
""" Obtain the current daemon state """
def get_state(self) -> str:
""" Upstream request: Obtain the current daemon state """
return NRMState(lib.state(self.commonOpts))
def workload_finished(self):
""" Checks NRM to see whether all tasks are finished. """
def all_finished(self) -> None:
""" Upstream request: Checks NRM to see whether all tasks are finished. """
return lib.finished(self.commonOpts)
def workload_recv(self):
""" Receive a message from NRM's upstream API. """
return self.upstreampub.recv()
def workload_action(self, actionList: List[Action]):
""" Send a message to NRM's upstream API. """
actionList = [(str(a.actuatorID), float(a.actuatorValue)) for a in actionList]
if lib.action(self.commonOpts, actionList):
pass
else:
raise (Exception("couldn't actuate"))
def workload_exit_status(self):
""" Check the workload's exit status. """
pass
def upstream_recv(self) -> {}:
""" Upstream listen: Receive a message from NRM's upstream API. """
return json.loads(self.upstreampub.recv())
......@@ -49,6 +49,7 @@ in mkShell {
withHoogle = true;
buildInputs = [
pkgs.git
pkgs.pythonPackages.mypy
pkgs.hwloc
pkgs.htop
pkgs.jq
......@@ -78,7 +79,6 @@ in mkShell {
] ++ lib.optional ihaskell [ pkgs.ihaskell ] ++ lib.optionals experiment [
pandoc
daemonize
pythonPackages.pandas
pythonPackages.matplotlib
pythonPackages.seaborn
......
......@@ -8,7 +8,7 @@ for file in ./examples/nrmd/*.dhall ; do
echo "merged file $rootname.yaml with defaults:"
yq -s '.[0] * .[1]' ./resources/defaults/nrmd.json ./examples/nrmd/${rootname}.yaml
echo "diffing with file: $file"
dhall diff " $file " " $(yq -s '.[0] * .[1]' ./resources/defaults/nrmd.json ./examples/nrmd/${rootname}.yaml | json-to-dhall ' (./hsnrm/hsnrm/dhall/types/nrmd.dhall).Cfg ' --records-loose) "
dhall diff " $file " " $(yq -s '.[0] * .[1]' ./resources/defaults/nrmd.json ./examples/nrmd/${rootname}.yaml | json-to-dhall ' (./hsnrm/hsnrm/dhall/types/nrmd.dhall).Cfg ' ) "
cat ./examples/nrmd/${rootname}.yaml | yq '' > ./examples/nrmd/${rootname}.json
done
......@@ -18,7 +18,7 @@ for file in ./examples/manifests/*.dhall ; do
echo "merged file $rootname.yaml with defaults:"
yq -s '.[0] * .[1]' ./resources/defaults/manifest.json ./examples/manifests/${rootname}.yaml
echo "diffing with file: $file"
dhall diff " $file " " $(yq -s '.[0] * .[1]' ./resources/defaults/manifest.json ./examples/manifests/${rootname}.yaml | json-to-dhall ' (./hsnrm/hsnrm/dhall/types/manifest.dhall).Manifest ' --records-loose) "
dhall diff " $file " " $(yq -s '.[0] * .[1]' ./resources/defaults/manifest.json ./examples/manifests/${rootname}.yaml | json-to-dhall ' (./hsnrm/hsnrm/dhall/types/manifest.dhall).Manifest ' ) "
cat ./examples/manifests/${rootname}.yaml | yq '' > ./examples/manifests/${rootname}.json
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment