configurator - support for derived parameter

scripts/codes_configurator.py

@@ -13,13 +13,6 @@ def main():
     # load the module
     mod = conf.import_from(args.substitute_py)
 
-    # checks - make sure cfields is set and is the correct type 
-    conf.check_cfields(mod)
-
-    # check that pairs are actually pairs
-    if len(args.token_pairs) % 2 != 0:
-        raise ValueError("token pairs must come in twos")
-
     # intitialize the configuration object 
     cfg = conf.configurator(mod, args.token_pairs)
 
@@ -44,6 +37,18 @@ def main():
 
     flog.close()
 
+sub_help = \
+'python file defining "cfields" variable consisting of a sequence of <name, ' \
+'value sequence> pairs. The following variables may optionally appear. ' \
+'"exceptions" - a sequence of dictionaries. Each configuration generated by ' \
+'"cfields" is checked against each dictionary in "exceptions"; if each value ' \
+'in the dict matches that of the configuration, then that configuration is ' \
+'skipped. "cfields_derived_labels", "cfields_derived" - the former is a ' \
+'sequence of strings identifying replace tokens that will be dynamically set ' \
+'based on the input configuration generated by cfields. The latter is a ' \
+'function that adds all <name,value> pairs for names in ' \
+'"cfields_derived_labels".'
+
 def parse_args():
     parser = argparse.ArgumentParser(\
             description="generate set of configuration files based on template "
@@ -51,13 +56,7 @@ def parse_args():
     parser.add_argument("template",
             help="template file with replace tokens")
     parser.add_argument("substitute_py",
-            help='python file defining "cfields" variable consisting of '
-                 'elements of the form '
-                 '( replacement_token, [replacements...]). An optional '
-                 'variable "exceptions" may also be provided, of the form '
-                 '( {replace_token : replacement, ...}, ...). This will '
-                 'exclude any configuration that exhibits the key/value '
-                 'pairs in any of the input dictionaries')
+            help=sub_help)
     parser.add_argument("token_pairs", nargs='*',
             help="a list of whitespace-separated token, replace pairs for "
                  "command-line-driven replacement (useful in shell scripts "

scripts/configurator.py

@@ -9,6 +9,12 @@ class configurator:
     # note: object assumes input has been validated by caller, though it will
     #       still throw if it finds duplicate keys in the dict
     def __init__(self, module, replace_pairs):
+        # checks - check cfields and friends
+        check_cfields(module)
+        # check that pairs are actually pairs
+        if len(replace_pairs) % 2 != 0:
+            raise ValueError("token pairs must come in twos")
+
         self.mod = module
         self.num_fields = len(self.mod.cfields)
         self.labels = [k[0] for k in self.mod.cfields] + replace_pairs[0::2]
@@ -16,6 +22,8 @@ class configurator:
         self.start_iter = False
         self.in_iter    = False
         self.has_except = "excepts" in self.mod.__dict__
+        self.has_derived = "cfields_derived_labels" in self.mod.__dict__ and \
+                           "cfields_derived" in self.mod.__dict__
 
         for i in range(0, len(replace_pairs), 2):
             k,vstr = replace_pairs[i], replace_pairs[i+1]
@@ -28,9 +36,13 @@ class configurator:
             if v == None:
                 v = vstr
             self.replace_map[k] = v
+            
+        # initialize derived labels if necessary
+        if self.has_derived:
+            self.labels += [l for l in self.mod.cfields_derived_labels]
+
 
     def __iter__(self):
-        # pre-generate an initial config and return self
         self.start_iter = True 
         self.in_iter = True
         return self
@@ -40,15 +52,11 @@ class configurator:
             raise StopIteration 
         elif self.start_iter:
             # first iteration - initialize the iterators
-            self.iterables   = [k[1].__iter__() for k in self.mod.cfields]
+            self.iterables = [k[1].__iter__() for k in self.mod.cfields]
             for i in range(0, self.num_fields):
                 v = self.iterables[i].next()
                 self.replace_map[self.labels[i]] = v
             self.start_iter = False
-            # check if this is a valid config, if not, then recurse
-            if self.has_except and is_replace_except(self.mod.excepts,
-                    self.replace_map):
-                return self.next()
         else:
             # > first iteration, perform the updates
             # generate the next config
@@ -66,6 +74,10 @@ class configurator:
             else:
                 # last iterable has finished, have generated full set
                 raise StopIteration 
+        # add derived fields before exceptions 
+        if self.has_derived:
+            self.mod.cfields_derived(self.replace_map)
+        # check if this is a valid config, if not, then recurse
         if self.has_except and is_replace_except(self.mod.excepts,
                 self.replace_map):
             return self.next()
@@ -110,6 +122,21 @@ def check_cfields(module):
             isinstance(module.excepts[0], dict)) :
         raise TypeError("excepts not in correct format, see usage")
 
+    dl = "cfields_derived_labels" in module.__dict__
+    d  = "cfields_derived" in module.__dict__
+    if (dl and not d) or (not dl and d):
+        raise TypeError("both cfields_derived_labels and cfields_derived must "
+                "be set")
+    elif dl and d and not \
+            (isinstance(module.cfields_derived_labels, Sequence) and \
+            isinstance(module.cfields_derived_labels[0], str) and \
+            hasattr(module.cfields_derived, "__call__")):
+        raise TypeError("cfields_derived_labels must be a sequence of "
+                "strings, cfields_derived must be callable (accepting a "
+                "dict of replace_token, replacement pairs and adding pairs "
+                "for each label in cfields_derived_labels")
+        
+
 
 # import a python file (assumes there is a .py suffix!!!)
 def import_from(filename):

scripts/example/example.template

@@ -29,7 +29,7 @@ PARAMS
    # - individual packet sizes for network operations
    #   (each "packet" is represented by an event)
    # - independent of underlying network being used
-   packet_size="512";
+   packet_size="C_PACKET_SZ";
    # - type of model to use (must match with corresponding LPGROUPS entry)
    modelnet="simplenet";
    # - model-specific parameters

scripts/example/params.py

@@ -2,4 +2,10 @@ cfields = ( ("C_NUM_SERVERS", [1<<i for i in range(1,3)]),
             ("C_NUM_REQS",    [1,2]),
             ("C_PAYLOAD_SZ",  [1024*i for i in range(1,3)]) )
 
+# example derived paramter - make the packet size 1/4 of the payload size
+cfields_derived_labels = ( "C_PACKET_SZ", )
+def cfields_derived(replace_map):
+    replace_map["C_PACKET_SZ"] = replace_map["C_PAYLOAD_SZ"] / 4
+
+# example exception - don't generate configs with these sets of params
 excepts = ( {"C_NUM_SERVERS" : 4, "C_PAYLOAD_SZ" : 1024}, )