Commit e47c3494 authored by Valentin Reis's avatar Valentin Reis

testing black

parent 476517f2
Pipeline #7816 failed with stages
in 2 minutes and 51 seconds
......@@ -4,9 +4,8 @@
rec {
nrm = pkgs.nrm;
hack = nrm.overrideAttrs (old:{
buildInputs = old.buildInputs ++ [
pkgs.python3Packages.flake8
pkgs.python3Packages.autopep8
pkgs.python3Packages.sphinx ];
propagatedBuildInputs = old.propagatedBuildInputs ++ [
pkgs.python37Packages.flake8
pkgs.python37Packages.black ];
});
}
......@@ -12,11 +12,11 @@
import logging
from schema import loadschema
logger = logging.getLogger('nrm')
logger = logging.getLogger("nrm")
def has(self, f):
return(f in self.app.keys())
return f in self.app.keys()
ImageManifest = loadschema("yml", "manifest")
......
......@@ -12,28 +12,30 @@ from __future__ import print_function
import logging
logger = logging.getLogger('nrm')
logger = logging.getLogger("nrm")
class Application(object):
"""Information about a downstream API user."""
thread_fsm_table = {'stable': {'i': 's_ask_i', 'd': 's_ask_d'},
's_ask_i': {'done': 'stable', 'noop': 'max'},
's_ask_d': {'done': 'stable', 'noop': 'min'},
'max': {'d': 'max_ask_d'},
'min': {'i': 'min_ask_i'},
'max_ask_d': {'done': 'stable', 'noop': 'noop'},
'min_ask_i': {'done': 'stable', 'noop': 'noop'},
'noop': {}}
thread_fsm_table = {
"stable": {"i": "s_ask_i", "d": "s_ask_d"},
"s_ask_i": {"done": "stable", "noop": "max"},
"s_ask_d": {"done": "stable", "noop": "min"},
"max": {"d": "max_ask_d"},
"min": {"i": "min_ask_i"},
"max_ask_d": {"done": "stable", "noop": "noop"},
"min_ask_i": {"done": "stable", "noop": "noop"},
"noop": {},
}
def __init__(self, uuid, container, progress, threads, phase_contexts):
self.uuid = uuid
self.container_uuid = container
self.progress = progress
self.threads = threads
self.thread_state = 'stable'
self.thread_state = "stable"
self.phase_contexts = phase_contexts
def do_thread_transition(self, event):
......@@ -49,21 +51,21 @@ class Application(object):
# TODO: not a real model
if command not in self.thread_fsm_table[self.thread_state]:
return 0.0
speed = float(self.progress)/float(self.threads['cur'])
if command == 'i':
speed = float(self.progress) / float(self.threads["cur"])
if command == "i":
return speed
else:
return -speed
def update_threads(self, msg):
"""Update the thread tracking."""
newth = msg['payload']
curth = self.threads['cur']
newth = msg["payload"]
curth = self.threads["cur"]
if newth == curth:
self.do_thread_transition('noop')
self.do_thread_transition("noop")
else:
self.do_thread_transition('done')
self.threads['cur'] = newth
self.do_thread_transition("done")
self.threads["cur"] = newth
def update_progress(self, msg):
"""Update the progress tracking."""
......@@ -75,10 +77,11 @@ class Application(object):
def update_phase_context(self, msg):
"""Update the phase contextual information."""
id = int(msg.cpu)
self.phase_contexts[id] = {k: getattr(msg, k) for k in
('aggregation', 'computetime',
'totaltime')}
self.phase_contexts[id]['set'] = True
self.phase_contexts[id] = {
k: getattr(msg, k)
for k in ("aggregation", "computetime", "totaltime")
}
self.phase_contexts[id]["set"] = True
class ApplicationManager(object):
......@@ -91,21 +94,22 @@ class ApplicationManager(object):
def register(self, msg, container):
"""Register a new downstream application."""
uuid = msg['application_uuid']
container_uuid = msg['container_uuid']
uuid = msg["application_uuid"]
container_uuid = msg["container_uuid"]
progress = 0
threads = False
phase_contexts = dict()
phase_context_keys = ['set', 'aggregation', 'computetime', 'totaltime']
if container.power['policy']:
phase_context_keys = ["set", "aggregation", "computetime", "totaltime"]
if container.power["policy"]:
ids = container.resources.cpus
for id in ids:
phase_contexts[id] = dict.fromkeys(phase_context_keys)
phase_contexts[id]['set'] = False
phase_contexts[id]["set"] = False
else:
phase_contexts = None
self.applications[uuid] = Application(uuid, container_uuid, progress,
threads, phase_contexts)
self.applications[uuid] = Application(
uuid, container_uuid, progress, threads, phase_contexts
)
def delete(self, uuid):
"""Delete an application from the register."""
......
This diff is collapsed.
......@@ -12,7 +12,7 @@ from __future__ import print_function
import logging
logger = logging.getLogger('nrm')
logger = logging.getLogger("nrm")
class Action(object):
......@@ -73,19 +73,20 @@ class PowerActuator(object):
actions = []
pl = self.sensor_manager.get_powerlimits()
logger.info("power limits: %r:", pl)
if target == 'i':
if target == "i":
for k in pl:
r = range(int(pl[k]['curW'])+1, int(pl[k]['maxW']))
r = range(int(pl[k]["curW"]) + 1, int(pl[k]["maxW"]))
actions.extend([Action(k, s, s - r[0]) for s in r])
elif target == 'd':
elif target == "d":
for k in pl:
r = range(1, int(pl[k]['curW']))
r = range(1, int(pl[k]["curW"]))
actions.extend([Action(k, s, r[-1] - s) for s in r])
return actions
def execute(self, action):
logger.info("changing power limit: %r, %r", action.command,
action.delta)
logger.info(
"changing power limit: %r, %r", action.command, action.delta
)
self.sensor_manager.set_powerlimit(action.target, action.command)
def update(self, action):
......@@ -102,17 +103,18 @@ class Controller(object):
def planify(self, target, machineinfo):
"""Plan the next action for the control loop."""
try:
total_power = machineinfo['energy']['power']['total']
total_power = machineinfo["energy"]["power"]["total"]
except TypeError:
logging.error("\"machineinfo\" malformed. Can not run "
"control loop.")
logging.error(
'"machineinfo" malformed. Can not run ' "control loop."
)
return (None, None)
direction = None
if total_power < target:
direction = 'i'
direction = "i"
elif total_power > target:
direction = 'd'
direction = "d"
if direction:
actions = []
......@@ -139,26 +141,29 @@ class Controller(object):
ids = container.resources.cpus
pcs = application.phase_contexts
# Run policy only if all phase contexts have been received
if not filter(lambda i: not pcs[i]['set'], ids):
if not filter(lambda i: not pcs[i]["set"], ids):
# Only run policy if all phase contexts are an
# aggregation of same number of phases
aggs = [pcs[i]['aggregation'] for i in ids]
aggs = [pcs[i]["aggregation"] for i in ids]
if aggs.count(aggs[0]) == len(aggs):
container.power['manager'].run_policy(pcs)
if filter(lambda i: pcs[i]['set'], ids):
container.power["manager"].run_policy(pcs)
if filter(lambda i: pcs[i]["set"], ids):
logger.debug("Phase context not reset %r", application)
else:
container.power['manager'].reset_all()
container.power["manager"].reset_all()
for i in ids:
pcs[i]['set'] = False
pcs[i]["set"] = False
def run_policy(self, containers):
"""Run policies on containers with policies set."""
for container in containers:
p = containers[container].power
if p['policy']:
if p["policy"]:
apps = self.actuators[0].application_manager.applications
if apps:
app = next(apps[a] for a in apps if apps[a].container_uuid
== container)
app = next(
apps[a]
for a in apps
if apps[a].container_uuid == container
)
self.run_policy_container(containers[container], app)
......@@ -52,19 +52,19 @@ class cpustatvals:
def parse(self):
self.d = {} # clear d's contents
self.d['time'] = time.time()
self.d["time"] = time.time()
with open(self.cpustatfn(self.cpuid)) as f:
while True:
l = f.readline()
if not l:
break
a = l.split()
if a[0] in ('id', 'aperf', 'mperf', 'pstate', 'tsc'):
if a[0] in ("id", "aperf", "mperf", "pstate", "tsc"):
self.d[a[0]] = int(a[1])
def pr(self):
for k in ('id', 'aperf', 'mperf', 'pstate', 'tsc'):
print('%s=%d ' % (k, self.d[k]))
for k in ("id", "aperf", "mperf", "pstate", "tsc"):
print("%s=%d " % (k, self.d[k]))
print()
def diff_u64(self, v1, v2): # v1 - v2
......@@ -73,33 +73,32 @@ class cpustatvals:
return (self.u64max - v2) + v1
def calc_cpufreq(self, prev): # prev is an object of cpustatvals
if not (prev.d.has_key('tsc') and self.d.has_key('tsc')):
if not (prev.d.has_key("tsc") and self.d.has_key("tsc")):
return 0.0
tmp = {}
for k in ('tsc', 'aperf', 'mperf'):
for k in ("tsc", "aperf", "mperf"):
tmp[k] = float(self.diff_u64(self.d[k], prev.d[k]))
dt = self.d['time'] - prev.d['time']
freq = tmp['aperf'] / tmp['mperf']
freq *= tmp['tsc']
dt = self.d["time"] - prev.d["time"]
freq = tmp["aperf"] / tmp["mperf"]
freq *= tmp["tsc"]
freq *= 1e-9 # covert it to GHz
freq /= dt
return freq
def calc_aperf(self, prev): # prev is an object of cpustatvals
if not (prev.d.has_key('tsc') and self.d.has_key('tsc')):
if not (prev.d.has_key("tsc") and self.d.has_key("tsc")):
return 0.0
tmp = {}
k = 'aperf'
k = "aperf"
tmp[k] = float(self.diff_u64(self.d[k], prev.d[k]))
dt = self.d['time'] - prev.d['time']
return tmp['aperf'] * 1e-9 / dt
dt = self.d["time"] - prev.d["time"]
return tmp["aperf"] * 1e-9 / dt
class cpufreq_reader:
def __init__(self):
self.outputpercore(True)
......@@ -122,7 +121,8 @@ class cpufreq_reader:
self.cnt = 0
self.samples = [
[cpustatvals(i) for i in self.cpus],
[cpustatvals(i) for i in self.cpus]]
[cpustatvals(i) for i in self.cpus],
]
self.sample()
......@@ -146,7 +146,7 @@ class cpufreq_reader:
if self.cnt % 2 == 0:
idx = 1
for cpuid in self.cpus:
ret[cpuid] = self.samples[idx][cpuid].d['pstate']
ret[cpuid] = self.samples[idx][cpuid].d["pstate"]
return ret
......@@ -165,7 +165,8 @@ class cpufreq_reader:
for cpuid in self.cpus:
ret[cpuid] = self.samples[idxcur][cpuid].calc_cpufreq(
self.samples[idxprev][cpuid])
self.samples[idxprev][cpuid]
)
return ret
......@@ -184,7 +185,8 @@ class cpufreq_reader:
for cpuid in self.cpus:
ret[cpuid] = self.samples[idxcur][cpuid].calc_aperf(
self.samples[idxprev][cpuid])
self.samples[idxprev][cpuid]
)
return ret
......@@ -193,7 +195,7 @@ class cpufreq_reader:
def sample_and_json(self, node=""):
if not self.init:
return ''
return ""
self.sample()
f = self.aperf()
......@@ -211,23 +213,23 @@ class cpufreq_reader:
if self.percore:
for c in self.ct.pkgcpus[p]:
buf += ',"c%d":%.3lf' % (c, f[c])
buf += '}'
buf += '}'
buf += "}"
buf += "}"
return buf
if __name__ == '__main__':
if __name__ == "__main__":
freq = cpufreq_reader()
if not freq.init:
print('Please check the cpustat module is installed')
print("Please check the cpustat module is installed")
sys.exit(1)
freq.outputpercore(False)
for i in range(0, 20):
j = freq.sample_and_json()
print('[freq json]')
print("[freq json]")
print(j)
time.sleep(1)
......@@ -236,23 +238,23 @@ if __name__ == '__main__':
for i in range(0, 20):
freq.sample()
print('[pstate]')
print("[pstate]")
for p in freq.pstate():
print(p)
print()
print('[aperf]')
print("[aperf]")
for f in freq.aperf():
print('%.2lf ' % f)
print("%.2lf " % f)
print()
print('[freq]')
print("[freq]")
for f in freq.cpufreq():
print('%.2lf ' % f)
print("%.2lf " % f)
print()
j = freq.sample_and_json()
print('[freq json]')
print("[freq json]")
print(j)
print()
......
......@@ -33,7 +33,7 @@ class coretemp_reader:
except:
return retval
l = f.readline()
m = re.search('Physical id ([0-9]+)', l)
m = re.search("Physical id ([0-9]+)", l)
if m:
retval = int(m.group(1))
f.close()
......@@ -46,19 +46,19 @@ class coretemp_reader:
except:
return retval
l = f.readline()
m = re.search('Core ([0-9]+)', l)
m = re.search("Core ([0-9]+)", l)
if m:
retval = int(m.group(1))
f.close()
return retval
hwmondir = '/sys/class/hwmon/'
hwmondir = "/sys/class/hwmon/"
class coretempinfo:
def __init__(self):
self.dir = ''
self.dir = ""
self.coretempfns = {} # use coreid as key
self.pkgtempfn = ''
self.pkgtempfn = ""
def __init__(self):
self.outputpercore(True)
......@@ -73,21 +73,23 @@ class coretemp_reader:
pkgid = -1
coretempfns = {}
pkgtempfn = ''
pkgtempfn = ""
# parse all temp*_label files
for d2 in os.listdir(tmpdir):
m = re.search('temp([0-9]+)_label', d2)
m = re.search("temp([0-9]+)_label", d2)
if m:
tempid = int(m.group(1))
coreid = self.parse_coretemp("%s/%s" % (tmpdir, d2))
if coreid >= 0:
coretempfns[coreid] = "%s/temp%d_input" % (
tmpdir, tempid)
tmpdir,
tempid,
)
else: # possibly pkgid
pkgtempfn = "%s/temp%d_input" % (tmpdir, tempid)
pkgid = self.parse_pkgtemp("%s/%s" % (tmpdir, d2))
if pkgid < 0:
print('unlikely: ', pkgtempfn)
print("unlikely: ", pkgtempfn)
cti = self.coretempinfo()
cti.dir = tmpdir
......@@ -105,11 +107,11 @@ class coretemp_reader:
for pkgid in sorted(ctemp.keys()):
temps = {}
if os.access(ctemp[pkgid].pkgtempfn, os.R_OK):
val = int(readbuf(ctemp[pkgid].pkgtempfn))/1000
temps['pkg'] = val
val = int(readbuf(ctemp[pkgid].pkgtempfn)) / 1000
temps["pkg"] = val
for c in sorted(ctemp[pkgid].coretempfns.keys()):
if os.access(ctemp[pkgid].coretempfns[c], os.R_OK):
val = int(readbuf(ctemp[pkgid].coretempfns[c]))/1000
val = int(readbuf(ctemp[pkgid].coretempfns[c])) / 1000
temps[c] = val
ret[pkgid] = temps
return ret
......@@ -126,10 +128,10 @@ class coretemp_reader:
key = "p%d" % p
ret[key] = dict()
pstat = self.getpkgstats(temp, p)
ret[key]['mean'] = pstat[0]
ret[key]['std'] = pstat[1]
ret[key]['min'] = pstat[2]
ret[key]['max'] = pstat[3]
ret[key]["mean"] = pstat[0]
ret[key]["std"] = pstat[1]
ret[key]["min"] = pstat[2]
ret[key]["max"] = pstat[3]
if self.percore:
for c in sorted(temp[p].keys()):
ret[key][c] = temp[p][c]
......@@ -151,7 +153,7 @@ class coretemp_reader:
def readpkgtemp(self):
fn = "%s_input" % self.pkgtempfns[pkgid].pkgfn
f = open(fn)
v = int(f.readline())/1000.0
v = int(f.readline()) / 1000.0
f.close()
return v
......@@ -162,7 +164,7 @@ class coretemp_reader:
if not os.access(fn, os.R_OK):
continue # cpu may become offline
f = open(fn)
v = int(f.readline())/1000.0
v = int(f.readline()) / 1000.0
f.close()
t.append(v)
return t
......@@ -172,7 +174,7 @@ class acpi_power_meter_reader:
# add a nicer detection routine later
def __init__(self):
self.init = False
fn = '/sys/class/hwmon/hwmon0/device/power1_average'
fn = "/sys/class/hwmon/hwmon0/device/power1_average"
if os.path.exists(fn):
self.init = True
......@@ -185,7 +187,7 @@ class acpi_power_meter_reader:
retval = -1
fn = '/sys/class/hwmon/hwmon0/device/power1_average'
fn = "/sys/class/hwmon/hwmon0/device/power1_average"
try:
f = open(fn, "r")
except:
......@@ -200,4 +202,4 @@ class acpi_power_meter_reader:
if not self.init:
return {}
return {'power': self.read()}
return {"power": self.read()}
......@@ -32,11 +32,11 @@ def readbuf(fn):
except:
time.sleep(0.01)
continue
return ''
return ""
def readuptime():
f = open('/proc/uptime')
f = open("/proc/uptime")
l = f.readline()
v = l.split()
return float(v[0])
......@@ -35,27 +35,27 @@ from clr_misc import *
class cputopology:
cpubasedir = '/sys/devices/system/cpu/'
nodebasedir = '/sys/devices/system/node/'
cpubasedir = "/sys/devices/system/cpu/"
nodebasedir = "/sys/devices/system/node/"
def parserange(self, fn):
tmp = readbuf(fn)
ret = []
for t in tmp.split(','):
ab = re.findall('[0-9]+', t)
for t in tmp.split(","):
ab = re.findall("[0-9]+", t)
if len(ab) == 2:
ret = ret + range(int(ab[0]), int(ab[1])+1)
ret = ret + range(int(ab[0]), int(ab[1]) + 1)
elif len(ab) == 1:
ret = ret + [int(ab[0])]
else:
print('unlikely at cputoplogy.parserange():', ab)
print("unlikely at cputoplogy.parserange():", ab)
sys.exit(1)
return ret
def parsemask(self, fn):
tmp = readbuf(fn)
tmp = tmp.rstrip()
maskstrs = tmp.split(',')
maskstrs = tmp.split(",")
maskstrs.reverse()
shift = 0
ret = []
......@@ -63,20 +63,22 @@ class cputopology:
bmint = long(mstr, 16)
for i in range(0, 32):
if (bmint & 1) == 1:
ret.append(i+shift)
ret.append(i + shift)
bmint = bmint >> 1
shift = shift + 32
return ret
def detect(self):
self.onlinecpus = self.parserange(self.cpubasedir + 'online')
self.onlinecpus = self.parserange(self.cpubasedir + "online")
self.pkgcpus = {}
for cpuid in self.onlinecpus:
pkgidfn = self.cpubasedir + \
"cpu%d/topology/physical_package_id" % (cpuid)
pkgidfn = (
self.cpubasedir
+ "cpu%d/topology/physical_package_id" % (cpuid)
)
pkgid = int(readbuf(pkgidfn))
if not self.pkgcpus.has_key(pkgid):
self.pkgcpus[pkgid] = []
......@@ -91,29 +93,29 @@ class cputopology:
self.cpu2coreid[cpuid] = (pkgid, coreid)
self.core2cpuid[(pkgid, coreid)] = cpuid
self.onlinenodes = self.parserange(self.nodebasedir + 'online')
self.onlinenodes = self.parserange(self.nodebasedir + "online")
self.nodecpus = {}
for n in self.onlinenodes:
self.nodecpus[n] = self.parsemask(
self.nodebasedir + "node%d/cpumap" % (n))
self.nodebasedir + "node%d/cpumap" % (n)
)
def __init__(self):
self.detect()
class nodeconfig:
def parse(self):
self.hostname = socket.gethostname()
# XXX: not sure this is unique
self.nodename = self.hostname.split('.')[0]
self.nodename = self.hostname.split(".")[0]
tmp = readbuf('/proc/version')
tmp = readbuf("/proc/version")
self.version = tmp.split()[2]
re_model = re.compile("^model\s+:\s+([0-9]+)")
self.cpumodel = -1
with open('/proc/cpuinfo') as f:
with open("/proc/cpuinfo") as f:
while True:
l = f.readline()
if not l:
......@@ -123,16 +125,16 @@ class nodeconfig:
self.cpumodel = int(m.group(1))
self.memoryKB = -1
with open('/proc/meminfo') as f:
with open("/proc/meminfo") as f:
l = f.readline()
self.memoryKB = int(l.split()[1])
# assume that all cpu have the same setting for this experiment
self.driver = ''
self.freqdriver = ''
d = '/sys/devices/system/cpu/cpu0/cpufreq'
self.driver = ""
self.freqdriver = ""
d = "/sys/devices/system/cpu/cpu0/cpufreq"
if os.path.exists(d):
self.freqdriver = 'acpi_cpufreq'
self.freqdriver = "acpi_cpufreq"
fn = d + "/scaling_driver"
self.driver = readbuf(fn).rstrip()
fn = d + "/scaling_governor"
......@@ -142,61 +144,61 @@ class nodeconfig:
d = "/sys/devices/system/cpu/intel_pstate"
if os.path.exists(d):
self.freqdriver = 'pstate'
k = 'max_perf_pct'
self.freqdriver = "pstate"
k = "max_perf_pct"
pmax = readbuf("%s/%s" % (d, k)).rstrip()
k = 'min_perf_pct'
k = "min_perf_pct"
pmin = readbuf("%s/%s" % (d, k)).rstrip()
k = 'no_turbo'
k = "no_turbo"
noturbo = readbuf("%s/%s" % (d, k)).rstrip()
self.pstate = "%s/%s/%s" % (pmax, pmin, noturbo)
d = "/sys/devices/system/cpu/turbofreq"
if os.path.exists(d):
self.freqdriver = 'coolrfreq'
self.policy = d + '/pstate_policy'
self.freqdriver = "coolrfreq"
self.policy = d + "/pstate_policy"
def __init__(self):
self.parse()
def testnodeconfig():
print('=== ', sys._getframe().f_code.co_name)
print("=== ", sys._getframe().f_code.co_name)
nc = nodeconfig()
print('node: ', nc.nodename)
print('version: ', nc.version)