filtering of auto-generated config files by fields

scripts/codes_configurator.py

 #!/usr/bin/python2.7
 
-import re
-import sys
 import os
-import imp
 import argparse
-from collections import Sequence
+import configurator as conf
 
 def main():
     args = parse_args()
@@ -14,123 +11,38 @@ def main():
     tstr = open(args.template).read()
 
     # load the module
-    mod = import_from(args.substitute_py)
+    mod = conf.import_from(args.substitute_py)
 
     # checks - make sure cfields is set and is the correct type 
-    if "cfields" not in mod.__dict__:
-        raise TypeError("Expected cfields to be defined in " + args.substitute_py)
-    elif not \
-            (isinstance(mod.cfields, Sequence) and \
-            isinstance(mod.cfields[0][0], str) and \
-            isinstance(mod.cfields[0][1], Sequence)):
-        raise TypeError("cfields in incorrect format, see usage")
-    num_fields = len(mod.cfields)
-    
-    # get list of iterators, replacement input, and output labels
-    labels      = [ k[0] for k in mod.cfields]
-    iterables   = [ k[1].__iter__() for k in mod.cfields]
-    replace_map =     { k[0] : None for k in mod.cfields }
+    conf.check_cfields(mod)
 
-    # process the command line token replacements, adding to labels
-    # and replace_map, but not to iterables
+    # check that pairs are actually pairs
     if len(args.token_pairs) % 2 != 0:
         raise ValueError("token pairs must come in twos")
-    elif len(args.token_pairs) > 0:
-        for i in range(0, len(args.token_pairs), 2):
-            k,vstr = args.token_pairs[i],args.token_pairs[i+1]
-            # try making v numeric - fall back to string
-            v = try_num(vstr)
-            if v == None:
-                v = vstr
-            # add pair to replace map and labels (but not iterables!)
-            if k in replace_map:
-                raise ValueError("token " + k + " of token_pairs matches a token "
-                        "given by the substitution py file")
-            else:
-                replace_map[k] = v
-                labels.append(k)
+
+    # intitialize the configuration object 
+    cfg = conf.configurator(mod, args.token_pairs)
 
     # print the header to the log
     if args.log != None:
         flog = open(args.log, "w")
     else:
         flog = open(os.devnull, "w")
-    flog.write("# format:\n# <config index>")
-    for l in labels:
-        flog.write(" <" + l + ">")
-    flog.write("\n")
-
-    # generate initial configuration 
-    for i in range(0,num_fields):
-        v = iterables[i].next()
-        replace_map[labels[i]] = v
+    
+    cfg.write_header(flog)
 
-    # main loop 
-    ct = 0
-    while True:
-        # generate new configuration
-        new_config = replace_many(tstr, replace_map)
-        fname = template+"."+str(ct) if args.output_prefix==None else \
-            args.output_prefix+"."+str(ct)
+    # main loop (cfg iterator returns nothing, eval'd for side effects)
+    for i,_ in enumerate(cfg):
+        new_config = conf.replace_many(tstr, cfg.replace_map)
+        fname = template+"."+str(i) if args.output_prefix==None else \
+            args.output_prefix+"."+str(i)
         with open(fname, "w") as fout:
             fout.write(new_config)
+        
+        # write this config to the log 
+        cfg.write_config_line(i, flog)
 
-        # print the configuration to the log
-        flog.write(str(ct))
-        for l in labels:
-            flog.write(' ' + str(replace_map[l]))
-        else:
-            flog.write('\n')
-
-        # generate the next config
-        for i in range(0,num_fields):
-            try:
-                # update current iterable and finish
-                v = iterables[i].next()
-                replace_map[labels[i]] = v
-                ct += 1
-                break
-            except StopIteration:
-                # reset the current iterable and set to first element
-                iterables[i] = mod.cfields[i][1].__iter__()
-                v = iterables[i].next()
-                replace_map[labels[i]] = v
-        else:
-            # last iterable has finished, have generated full set
-            break
-
-# import a python file (assumes there is a .py suffix!!!)
-def import_from(filename):
-    path,name = os.path.split(filename)
-    name,ext  = os.path.splitext(name) 
-
-    fmod, fname, data = imp.find_module(name, [path])
-    return imp.load_module(name,fmod,fname,data)
-
-# given a string and a set of substitution pairs, 
-#   perform a single pass replace 
-# st is the string to perform substitution on
-# kv_pairs is a dict or a sequence of sequences.
-# kv_pairs examples:
-# { a:b, c:d }
-# [[a,b],[c,d]]
-# [(a,b),(c,d)]
-# ((a,b),(c,d))
-def replace_many(st, kv_pairs):
-    rep_dict = {}
-    # dict-ify and string-ify the input
-    if isinstance(kv_pairs, dict):
-        for k in kv_pairs:
-            rep_dict[k] = str(kv_pairs[k])
-    elif isinstance(kv_pairs, Sequence) and isinstance(kv_pairs[0], Sequence):
-        for k in kv_pairs:
-            rep_dict[k[0]] = str(kv_pairs[k[1]])
-    else:
-        raise TypeError("Expected dict or sequence of sequences types")
-
-    pat = re.compile("|".join([re.escape(k) for k in rep_dict]))
-
-    return pat.sub(lambda match: rep_dict[match.group(0)], st)
+    flog.close()
 
 def parse_args():
     parser = argparse.ArgumentParser(\
@@ -142,8 +54,7 @@ def parse_args():
             help='python file defining "cfields" variable consisting of '
                  'elements of the form '
                  '( replacement_token, [replacements...])')
-    parser.add_argument("token_pairs",
-            nargs='*',
+    parser.add_argument("token_pairs", nargs='*',
             help="a list of whitespace-separated token, replace pairs for "
                  "command-line-driven replacement (useful in shell scripts "
                  "for generating sets of configuration files with a distinct "
@@ -155,14 +66,5 @@ def parse_args():
                  "(default: the configuration index appended to the template name)")
     return parser.parse_args()
 
-def try_num(s):
-    try:
-        return int(s)
-    except ValueError:
-        try:
-            return float(s)
-        except ValueError:
-            return None
-
 if __name__ == "__main__":
     main()

scripts/codes_filter_configs.py

+#!/usr/bin/python2.7
+
+import configurator as conf
+from sys import stdout
+import argparse
+
+def main():
+    args = parse_args()
+
+    # load and check the module
+    mod = conf.import_from(args.substitute_py)
+    conf.check_cfields(mod)
+
+    # check that pairs are actually pairs
+    tp = args.token_pairs
+    if len(tp) % 2 != 0:
+        raise ValueError("token pairs must come in twos")
+
+    # intitialize the configuration object 
+    cfg = conf.configurator(mod, [])
+
+    # build a lookup structure for the pairs, converting to numbers if 
+    # possible
+    tdict = { }
+    for i in range(0, len(tp), 2):
+        v = conf.try_num(tp[i+1])
+        if v == None:
+            v = tp[i+1]
+        tdict[tp[i]] = v
+
+    # quick check - make sure pairs are actually in the config
+    for k in tdict:
+        if k not in cfg.replace_map:
+            raise ValueError("key " + k + " not in original config")
+
+    # filter out the files
+    # (cfg iterator returns nothing, eval'd for side effects)
+    for i,_ in enumerate(cfg):
+        for k in tdict:
+            if tdict[k] != cfg.replace_map[k]:
+                break
+        else:
+            stdout.write(args.output_prefix + "." + str(i) + "\n")
+
+def parse_args():
+    parser = argparse.ArgumentParser(\
+            description="emit configuration files to stdout matching given "
+            "token values")
+    parser.add_argument("output_prefix", help="prefix of output config files")
+    parser.add_argument("substitute_py", 
+            help="input python file given to codes_configurator.py - see "
+            "codes_configurator.py for details")
+    parser.add_argument("token_pairs", nargs='*',
+            help="a list of whitespace-separated token, replace pairs to "
+            "filter the set of generated configuration files by")
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    main()

scripts/configurator.py

+""" Helpers and utilities for use by codes_configurator and friends """
+
+import re
+import os
+import imp
+from collections import Sequence
+
+class configurator:
+    # note: object assumes input has been validated by caller, though it will
+    #       still throw if it finds duplicate keys in the dict
+    def __init__(self, module, replace_pairs):
+        self.mod = module
+        self.num_fields = len(self.mod.cfields)
+        self.labels = [k[0] for k in self.mod.cfields] + replace_pairs[0::2]
+        self.replace_map = { k[0] : None for k in self.mod.cfields }
+        self.start_iter = False
+        self.in_iter    = False
+
+        for i in range(0, len(replace_pairs), 2):
+            k,vstr = replace_pairs[i], replace_pairs[i+1]
+            # add pair to replace map and labels (but not iterables!)
+            if k in self.replace_map:
+                raise ValueError("token " + k + " of token_pairs matches a token "
+                        "given by the substitution py file")
+            # try making v numeric - fall back to string
+            v = try_num(vstr)
+            if v == None:
+                v = vstr
+            self.replace_map[k] = v
+
+    def __iter__(self):
+        # pre-generate an initial config and return self
+        self.start_iter = True 
+        self.in_iter = True
+        return self
+    def next(self):
+        if not self.in_iter:
+            # either uninitialized or at the end of the road
+            raise StopIteration 
+        elif self.start_iter:
+            # first iteration - initialize the iterators
+            self.iterables   = [k[1].__iter__() for k in self.mod.cfields]
+            for i in range(0, self.num_fields):
+                v = self.iterables[i].next()
+                self.replace_map[self.labels[i]] = v
+            self.start_iter = False
+        else:
+            # > first iteration, perform the updates
+            # generate the next config
+            for i in range(0,self.num_fields):
+                try:
+                    # update current iterable and finish
+                    v = self.iterables[i].next()
+                    self.replace_map[self.labels[i]] = v
+                    break
+                except StopIteration:
+                    # reset the current iterable and set to first element
+                    self.iterables[i] = self.mod.cfields[i][1].__iter__()
+                    v = self.iterables[i].next()
+                    self.replace_map[self.labels[i]] = v
+            else:
+                # last iterable has finished, have generated full set
+                raise StopIteration 
+        return None
+
+    def write_header(self, fout):
+        fout.write("# format:\n# <config index>")
+        for l in self.labels:
+            fout.write(" <" + l + ">")
+        fout.write("\n")
+
+    def write_config_line(self, ident, fout):
+        # print the configuration to the log
+        fout.write(str(ident))
+        for l in self.labels:
+            fout.write(' ' + str(self.replace_map[l]))
+        else:
+            fout.write('\n')
+
+# checks - make sure cfields is set and is the correct type 
+def check_cfields(module):
+    if "cfields" not in module.__dict__:
+        raise TypeError("Expected cfields to be defined in " + str(module))
+    elif not \
+            (isinstance(module.cfields, Sequence) and \
+            isinstance(module.cfields[0][0], str) and \
+            isinstance(module.cfields[0][1], Sequence)):
+        raise TypeError("cfields in incorrect format, see usage")
+
+# import a python file (assumes there is a .py suffix!!!)
+def import_from(filename):
+    path,name = os.path.split(filename)
+    name,ext  = os.path.splitext(name) 
+
+    fmod, fname, data = imp.find_module(name, [path])
+    return imp.load_module(name,fmod,fname,data)
+
+# attempts casting to a numeric type, returning None on failure
+def try_num(s):
+    try:
+        return int(s)
+    except ValueError:
+        try:
+            return float(s)
+        except ValueError:
+            return None
+
+# given a string and a set of substitution pairs, 
+#   perform a single pass replace 
+# st is the string to perform substitution on
+# kv_pairs is a dict or a sequence of sequences.
+# kv_pairs examples:
+# { a:b, c:d }
+# [[a,b],[c,d]]
+# [(a,b),(c,d)]
+# ((a,b),(c,d))
+def replace_many(st, kv_pairs):
+    rep_dict = {}
+    # dict-ify and string-ify the input
+    if isinstance(kv_pairs, dict):
+        for k in kv_pairs:
+            rep_dict[k] = str(kv_pairs[k])
+    elif isinstance(kv_pairs, Sequence) and isinstance(kv_pairs[0], Sequence):
+        for k in kv_pairs:
+            rep_dict[k[0]] = str(kv_pairs[k[1]])
+    else:
+        raise TypeError("Expected dict or sequence of sequences types")
+
+    pat = re.compile("|".join([re.escape(k) for k in rep_dict]))
+
+    return pat.sub(lambda match: rep_dict[match.group(0)], st)