Merge branch 'power_policy_interface' into 'master'

Merge Power policy interface

Closes #11

See merge request !18

nrm/containers.py

@@ -50,6 +50,8 @@ class ContainerManager(object):
         environ['AC_APP_NAME'] = manifest.name
         environ['AC_METADATA_URL'] = "localhost"
         logger.info("run: environ: %r", environ)
+        # TODO: Application library to load must be set during configuration
+        applicationpreloadlibrary = '.so'
 
         # create container
         container_name = request['uuid']
@@ -74,6 +76,12 @@ class ContainerManager(object):
                 if manifest.app.isolators.perfwrapper.enabled in ["1", "True"]:
                     argv.append('argo-perf-wrapper')
 
+        if hasattr(manifest.app.isolators, 'powerpolicy'):
+            if hasattr(manifest.app.isolators.powerpolicy, 'enabled'):
+                if manifest.app.isolators.powerpolicy.enabled in ["1", "True"]:
+                    if manifest.app.isolators.powerpolicy.policy != "NONE":
+                        environ['LD_PRELOAD'] = applicationpreloadlibrary
+
         argv.append(command)
         argv.extend(args)
         process = self.nodeos.execute(container_name, argv, environ)

nrm/coolr/dutycycle.py

+""" DutyCycle Module:
+    This module contains functions to set duty cyle of each cpu in intervals
+    of 6.25% min = 6.25% max(default) = 100%
+
+    Value   -   Duty Cycle
+    0       -       100.0%
+    1       -        6.25%
+    2       -        12.5%
+    3       -       18.75%
+    .
+    .
+    .
+    15      -       93.75%
+    16      -       100.0%
+
+    There are also functions to reset the duty cycle of a cpu and check the
+    current value of duty cycle
+"""
+
+import msr
+
+
+class DutyCycle:
+
+    def __init__(self):
+        self.register = 0x19A
+        self.msr = msr.Msr()
+
+    # set duty cycle of a cpu
+    def set(self, cpu, value):
+        if 0 < value < 16:
+            self.msr.write(cpu, self.register, 16 + value)
+        else:
+            self.msr.write(cpu, self.register, 0)
+
+    # reset duty cycle of a cpu
+    def reset(self, cpu):
+        self.msr.write(cpu, self.register, 0)
+
+    # check current duty cycle value
+    def check(self, cpu):
+        return self.msr.read(cpu, self.register)

nrm/coolr/msr.py

+""" Msr Module:
+    This module provides the interfaces to read and write msr through msr_safe
+    kernel module.
+
+    Note: msr_safe kernel module needs to be installed on your machine for this
+    module to work. To run with root privileges change 'msr_safe' in
+    get_file_name() function to 'msr'.
+"""
+
+import os
+import sys
+import errno
+import struct
+
+
+class Msr:
+    # get msr file name for the cpu
+    def get_file_name(self, cpu):
+        return '/dev/cpu/%d/msr_safe' % cpu
+
+    # open msr file with correct privileges
+    def file_open(self, filename, privilege):
+        try:
+            if privilege == 'r':
+                fd = os.open(filename, os.O_RDONLY)
+            elif privilege == 'w':
+                fd = os.open(filename, os.O_WRONLY)
+
+        except OSError as e:
+            if e.errno == errno.ENXIO:
+                sys.exit('file_open: No such device or address ' + filename)
+            elif e.errno == errno.EIO:
+                sys.exit('file_open: I/O error ' + filename)
+            elif e.errno == errno.EACCES:
+                sys.exit('file_open: Permission denied ' + filename)
+            else:
+                sys.exit('file_open: Error ' + filename)
+
+        return fd
+
+    # read a msr
+    def read(self, cpu, register):
+        msrfile = self.get_file_name(cpu)
+        fd = self.file_open(msrfile, 'r')
+        try:
+            os.lseek(fd, int(register), os.SEEK_SET)
+            """ read and handle binary data from msr file """
+            value = struct.unpack('Q', os.read(fd, 8))[0]
+            os.close(fd)
+
+        except OSError as e:
+            os.close(fd)
+            if e.errno == errno.EIO:
+                sys.exit('read: I/O error ' + msrfile)
+            elif e.errno == errno.EACCES:
+                sys.exit('read: Permission denied ' + msrfile)
+            else:
+                sys.exit('read: Error ' + msrfile)
+
+        return value
+
+    # write a msr
+    def write(self, cpu, register, value):
+        msrfile = self.get_file_name(cpu)
+        fd = self.file_open(msrfile, 'w')
+        try:
+            os.lseek(fd, int(register), os.SEEK_SET)
+            """ write binary data to msr file """
+            os.write(fd, struct.pack('Q', value))
+            os.close(fd)
+
+        except OSError as e:
+            os.close(fd)
+            if e.errno == errno.EIO:
+                sys.exit('write: I/O error ' + msrfile)
+            elif e.errno == errno.EACCES:
+                sys.exit('write: Permission denied ' + msrfile)
+            else:
+                sys.exit('write: Error ' + msrfile)
+
+        return value

nrm/ddcmpolicy.py

+""" DDCMPolicy Module:
+    This module contains the Dynamic Duty Cycle Modulation (DDCM) based policy
+    aimed at mitigating workload imbalance in parallel applications that use
+    barrier synchronizations. It reduces duty cycle of cpus not on the critical
+    path of execution thereby reducing energy with little or no adverse impact
+    on performance.
+
+    This implementation specifically targets Intel architecture.
+
+    Please check your architecture specification for supported power control
+    mechanisms and other information.
+
+    Additional information:
+
+    1. Bhalachandra, Sridutt, Allan Porterfield, and Jan F. Prins. "Using
+    dynamic duty cycle modulation to improve energy efficiency in high
+    performance computing." In Parallel and Distributed Processing Symposium
+    Workshop (IPDPSW), 2015 IEEE International, pp. 911-918. IEEE, 2015.
+
+    2. Porterfield, Allan, Rob Fowler, Sridutt Bhalachandra, Barry Rountree,
+    Diptorup Deb, and Rob Lewis. "Application runtime variability and power
+    optimization for exascale computers." In Proceedings of the 5th
+    International Workshop on Runtime and Operating Systems for Supercomputers,
+    p. 3. ACM, 2015.
+"""
+
+import math
+import coolr
+import coolr.dutycycle
+
+
+class DDCMPolicy:
+    """ Contains cpu-specific DDCM based power policy """
+    def __init__(self, maxlevel=16, minlevel=1):
+        self.maxdclevel = maxlevel
+        self.mindclevel = minlevel
+        # Relaxation factor
+        self.relaxation = 1
+
+        self.ddcmpolicyset = 0
+        self.ddcmpolicyreset = 0
+
+        self.dc = coolr.dutycycle.DutyCycle()
+
+    def print_stats(self, resetflag=False):
+        print('DDCM Policy: DDCMPolicySets %d DDCMPolicyResets %d' %
+              (self.ddcmpolicyset, self.ddcmpolicyreset))
+        if resetflag:
+            self.ddcmpolicyset = 0
+            self.ddcmpolicyreset = 0
+
+    def execute(self, cpu, currentdclevel, computetime, totalphasetime):
+        # Compute work done by cpu during current phase
+        work = computetime / totalphasetime
+
+        # Compute effective work based on current duty cycle(dc) level
+        effectivework = work * self.maxdclevel / currentdclevel
+
+        # Compute effective slow down in current phase
+        effectiveslowdown = work * self.mindclevel / currentdclevel
+
+        # Decrease or keep constant dc level in the next phase if the effective
+        # work done is equal or less than 1.0
+        if effectivework <= 1.0:
+            self.ddcmpolicyset += 1
+
+            # Compute by how many levels dc needs to decrease
+            dcreduction = math.floor(effectivework / 0.0625) - 15
+
+            # Compute new dc level for next phase
+            if -14 < dcreduction < 0:
+                # Note that dcreduction is a negative value
+                newdclevel = currentdclevel + dcreduction + self.relaxation
+            elif dcreduction < -13:
+                # Empirical observation shows reducing dc below 18.75% leads to
+                # excessive slowdown
+                newdclevel = currentdclevel - 13
+            else:
+                # If reduction required is 0
+                newdclevel = currentdclevel
+
+            # Check if new dc level computed is not less than whats permissible
+            if newdclevel < self.mindclevel:
+                newdclevel = self.maxdclevel
+
+        # If there was a slowdown in the last phase, then increase the duty
+        # cycle level corresponding to the slowdown
+        else:
+            self.ddcmpolicyreset += 1
+
+            # Compute by how many levels dc needs to increase
+            dcincrease = math.floor(effectiveslowdown / 0.0625)
+
+            newdclevel = currentdclevel + dcincrease
+
+            # Check if new dc level computed is not greater than whats
+            # permissible
+            if newdclevel > self.maxdclevel:
+                newdclevel = self.maxdclevel
+
+        # Set the duty cycle of cpu to the new value computed
+        self.dc.set(cpu, newdclevel)
+
+        return newdclevel

nrm/powerpolicy.py

+""" Power Policy Module:
+    This module provides the interfaces that enable use of policies to control
+    processor power using controls available in the processor.
+    E.g. Dynamic Duty Cycle Modulation (DDCM), Dynamic Voltage
+    and Frequency Scaling (DVFS) and Power capping
+
+    The policies target problems like workload imbalance, memory saturation
+    seen very often in parallel applications.
+
+    To mitigate workload imbalance the policies adapt core frequencies to
+    workload characteristics through use of core-specific power controls.
+    The user can choose from three policies - DDCM, DVFS and a combination of
+    DVFS and DDCM to mitiage  workload imbalance in parallel applications that
+    use barrier synchronizations.
+    The effective frequency of cpus not on the critical path of execution is
+    reduced thereby lowering energy with little or no adverse impact on
+    performance.
+
+    Additional information:
+
+    Bhalachandra, Sridutt, Allan Porterfield, Stephen L. Olivier, and Jan F.
+    Prins. "An adaptive core-specific runtime for energy efficiency." In 2017
+    IEEE International Parallel and Distributed Processing Symposium (IPDPS),
+    pp. 947-956. 2017.
+
+    Note: Power controls (DVFS, DDCM and power capping) needs to be enabled
+    before using these interfaces. Please check your architecture specification
+    for supported power contols and related information.
+"""
+import ddcmpolicy
+
+
+class PowerPolicyManager:
+    """ Used for power policy application """
+
+    def __init__(self, ncpus=0, policy='NONE', damper=0.1, slowdown=1.1):
+        self.policy = policy
+        self.damper = damper
+        self.slowdown = slowdown
+
+        # TODO: Need to set this based on container configuration
+        self.ncpus = ncpus
+
+        # Intiliaze all power interfaces
+        self.ddcmpolicy = ddcmpolicy.DDCMPolicy()
+
+        # Power levels
+        self.maxdclevel = self.ddcmpolicy.maxdclevel
+        # TODO: Need to set this value when DVFS policies are added
+        self.maxfreqlevel = -1
+        # TODO: Need to only allow power changes to cpus in container
+        self.dclevel = dict.fromkeys(range(0, self.ncpus), self.maxdclevel)
+        self.freqlevel = dict.fromkeys(range(0, self.ncpus), self.maxfreqlevel)
+
+        # Book-keeping
+        self.damperexits = 0
+        self.slowdownexits = 0
+        self.prevtolalphasetime = 10000.0   # Random large value
+
+    def run_policy(self, cpu, startcompute, endcompute, startbarrier,
+                   endbarrier):
+        # Select and invoke appropriate power policy
+        # TODO: Need to add a better policy selection logic in addition to user
+        # specified
+        ret, value = self.invoke_policy(cpu, self.policy, self.dclevel[cpu],
+                                        self.freqlevel[cpu], startcompute,
+                                        endcompute, startbarrier, endbarrier)
+        if self.policy == 'DDCM' and ret in ['DDCM', 'SLOWDOWN']:
+            self.dclevel[cpu] = value
+
+    def invoke_policy(self, cpu, policy, dclevel, freqlevel, startcompute,
+                      endcompute, startbarrier, endbarrier):
+        # Run with no policy
+        if policy == "NONE":
+            return 'NONE', -1
+
+        # Calculate time spent in computation, barrier in current phase along
+        # with total phase time
+        computetime = endcompute - startcompute
+        barriertime = endbarrier - startbarrier
+        totalphasetime = computetime + barriertime
+
+        # If the current phase length is less than the damper value, then do
+        # not use policy. This avoids use of policy during startup operation
+        # insignificant phases
+        if totalphasetime < self.damper:
+            self.damperexits += 1
+            return 'DAMPER', -1
+
+        # Reset value for next phase
+        self.prevtolalphasetime = totalphasetime
+
+        # If the current phase has slowed down beyond the threshold set, then
+        # reset power. This helps correct error in policy application or acts
+        # as a rudimentary way to detect phase change
+        if(dclevel < self.ddcmpolicy.maxdclevel and totalphasetime >
+                self.slowdown * self.prevtolalphasetime):
+            self.ddcmpolicy.dc.reset(cpu)
+            newdclevel = self.ddcmpolicy.maxdclevel
+
+            return 'SLOWDOWN', newdclevel
+
+        # Invoke the correct policy based on operation module
+        if policy == "DDCM":
+            newdclevel = self.ddcmpolicy.execute(cpu, dclevel, computetime,
+                                                 totalphasetime)
+
+        # TODO: Add DVFS and Combined policies
+
+            return 'DDCM', newdclevel
+
+    def print_policy_stats(self, resetflag=False):
+        # Get statistics for policy run
+        print('PowerPolicyManager: DamperExits %d SlowdownExits %d' %
+              (self.damperexits, self.slowdownexits))
+        self.ddcmpolicy.print_stats(resetflag)
+
+        if resetflag:
+            self.damperexits = 0
+            self.slowdownexits = 0
+
+    def power_reset(self, cpu):
+        # Reset all power controls
+        self.ddcmpolicy.dc.reset(cpu)
+
+        self.dclevel[cpu] = self.maxdclevel
+
+    def power_check(self, cpu):
+        # Check status of all power controls
+        return self.ddcmpolicy.dc.check(cpu)