Commit da5b6b71 authored by Jonathan Jenkins's avatar Jonathan Jenkins

filtering of auto-generated config files by fields

parent 3c968fb1
#!/usr/bin/python2.7
import re
import sys
import os
import imp
import argparse
from collections import Sequence
import configurator as conf
def main():
args = parse_args()
......@@ -14,123 +11,38 @@ def main():
tstr = open(args.template).read()
# load the module
mod = import_from(args.substitute_py)
mod = conf.import_from(args.substitute_py)
# checks - make sure cfields is set and is the correct type
if "cfields" not in mod.__dict__:
raise TypeError("Expected cfields to be defined in " + args.substitute_py)
elif not \
(isinstance(mod.cfields, Sequence) and \
isinstance(mod.cfields[0][0], str) and \
isinstance(mod.cfields[0][1], Sequence)):
raise TypeError("cfields in incorrect format, see usage")
num_fields = len(mod.cfields)
# get list of iterators, replacement input, and output labels
labels = [ k[0] for k in mod.cfields]
iterables = [ k[1].__iter__() for k in mod.cfields]
replace_map = { k[0] : None for k in mod.cfields }
conf.check_cfields(mod)
# process the command line token replacements, adding to labels
# and replace_map, but not to iterables
# check that pairs are actually pairs
if len(args.token_pairs) % 2 != 0:
raise ValueError("token pairs must come in twos")
elif len(args.token_pairs) > 0:
for i in range(0, len(args.token_pairs), 2):
k,vstr = args.token_pairs[i],args.token_pairs[i+1]
# try making v numeric - fall back to string
v = try_num(vstr)
if v == None:
v = vstr
# add pair to replace map and labels (but not iterables!)
if k in replace_map:
raise ValueError("token " + k + " of token_pairs matches a token "
"given by the substitution py file")
else:
replace_map[k] = v
labels.append(k)
# intitialize the configuration object
cfg = conf.configurator(mod, args.token_pairs)
# print the header to the log
if args.log != None:
flog = open(args.log, "w")
else:
flog = open(os.devnull, "w")
flog.write("# format:\n# <config index>")
for l in labels:
flog.write(" <" + l + ">")
flog.write("\n")
# generate initial configuration
for i in range(0,num_fields):
v = iterables[i].next()
replace_map[labels[i]] = v
cfg.write_header(flog)
# main loop
ct = 0
while True:
# generate new configuration
new_config = replace_many(tstr, replace_map)
fname = template+"."+str(ct) if args.output_prefix==None else \
args.output_prefix+"."+str(ct)
# main loop (cfg iterator returns nothing, eval'd for side effects)
for i,_ in enumerate(cfg):
new_config = conf.replace_many(tstr, cfg.replace_map)
fname = template+"."+str(i) if args.output_prefix==None else \
args.output_prefix+"."+str(i)
with open(fname, "w") as fout:
fout.write(new_config)
# write this config to the log
cfg.write_config_line(i, flog)
# print the configuration to the log
flog.write(str(ct))
for l in labels:
flog.write(' ' + str(replace_map[l]))
else:
flog.write('\n')
# generate the next config
for i in range(0,num_fields):
try:
# update current iterable and finish
v = iterables[i].next()
replace_map[labels[i]] = v
ct += 1
break
except StopIteration:
# reset the current iterable and set to first element
iterables[i] = mod.cfields[i][1].__iter__()
v = iterables[i].next()
replace_map[labels[i]] = v
else:
# last iterable has finished, have generated full set
break
# import a python file (assumes there is a .py suffix!!!)
def import_from(filename):
path,name = os.path.split(filename)
name,ext = os.path.splitext(name)
fmod, fname, data = imp.find_module(name, [path])
return imp.load_module(name,fmod,fname,data)
# given a string and a set of substitution pairs,
# perform a single pass replace
# st is the string to perform substitution on
# kv_pairs is a dict or a sequence of sequences.
# kv_pairs examples:
# { a:b, c:d }
# [[a,b],[c,d]]
# [(a,b),(c,d)]
# ((a,b),(c,d))
def replace_many(st, kv_pairs):
rep_dict = {}
# dict-ify and string-ify the input
if isinstance(kv_pairs, dict):
for k in kv_pairs:
rep_dict[k] = str(kv_pairs[k])
elif isinstance(kv_pairs, Sequence) and isinstance(kv_pairs[0], Sequence):
for k in kv_pairs:
rep_dict[k[0]] = str(kv_pairs[k[1]])
else:
raise TypeError("Expected dict or sequence of sequences types")
pat = re.compile("|".join([re.escape(k) for k in rep_dict]))
return pat.sub(lambda match: rep_dict[match.group(0)], st)
flog.close()
def parse_args():
parser = argparse.ArgumentParser(\
......@@ -142,8 +54,7 @@ def parse_args():
help='python file defining "cfields" variable consisting of '
'elements of the form '
'( replacement_token, [replacements...])')
parser.add_argument("token_pairs",
nargs='*',
parser.add_argument("token_pairs", nargs='*',
help="a list of whitespace-separated token, replace pairs for "
"command-line-driven replacement (useful in shell scripts "
"for generating sets of configuration files with a distinct "
......@@ -155,14 +66,5 @@ def parse_args():
"(default: the configuration index appended to the template name)")
return parser.parse_args()
def try_num(s):
try:
return int(s)
except ValueError:
try:
return float(s)
except ValueError:
return None
if __name__ == "__main__":
main()
#!/usr/bin/python2.7
import configurator as conf
from sys import stdout
import argparse
def main():
args = parse_args()
# load and check the module
mod = conf.import_from(args.substitute_py)
conf.check_cfields(mod)
# check that pairs are actually pairs
tp = args.token_pairs
if len(tp) % 2 != 0:
raise ValueError("token pairs must come in twos")
# intitialize the configuration object
cfg = conf.configurator(mod, [])
# build a lookup structure for the pairs, converting to numbers if
# possible
tdict = { }
for i in range(0, len(tp), 2):
v = conf.try_num(tp[i+1])
if v == None:
v = tp[i+1]
tdict[tp[i]] = v
# quick check - make sure pairs are actually in the config
for k in tdict:
if k not in cfg.replace_map:
raise ValueError("key " + k + " not in original config")
# filter out the files
# (cfg iterator returns nothing, eval'd for side effects)
for i,_ in enumerate(cfg):
for k in tdict:
if tdict[k] != cfg.replace_map[k]:
break
else:
stdout.write(args.output_prefix + "." + str(i) + "\n")
def parse_args():
parser = argparse.ArgumentParser(\
description="emit configuration files to stdout matching given "
"token values")
parser.add_argument("output_prefix", help="prefix of output config files")
parser.add_argument("substitute_py",
help="input python file given to codes_configurator.py - see "
"codes_configurator.py for details")
parser.add_argument("token_pairs", nargs='*',
help="a list of whitespace-separated token, replace pairs to "
"filter the set of generated configuration files by")
return parser.parse_args()
if __name__ == "__main__":
main()
""" Helpers and utilities for use by codes_configurator and friends """
import re
import os
import imp
from collections import Sequence
class configurator:
# note: object assumes input has been validated by caller, though it will
# still throw if it finds duplicate keys in the dict
def __init__(self, module, replace_pairs):
self.mod = module
self.num_fields = len(self.mod.cfields)
self.labels = [k[0] for k in self.mod.cfields] + replace_pairs[0::2]
self.replace_map = { k[0] : None for k in self.mod.cfields }
self.start_iter = False
self.in_iter = False
for i in range(0, len(replace_pairs), 2):
k,vstr = replace_pairs[i], replace_pairs[i+1]
# add pair to replace map and labels (but not iterables!)
if k in self.replace_map:
raise ValueError("token " + k + " of token_pairs matches a token "
"given by the substitution py file")
# try making v numeric - fall back to string
v = try_num(vstr)
if v == None:
v = vstr
self.replace_map[k] = v
def __iter__(self):
# pre-generate an initial config and return self
self.start_iter = True
self.in_iter = True
return self
def next(self):
if not self.in_iter:
# either uninitialized or at the end of the road
raise StopIteration
elif self.start_iter:
# first iteration - initialize the iterators
self.iterables = [k[1].__iter__() for k in self.mod.cfields]
for i in range(0, self.num_fields):
v = self.iterables[i].next()
self.replace_map[self.labels[i]] = v
self.start_iter = False
else:
# > first iteration, perform the updates
# generate the next config
for i in range(0,self.num_fields):
try:
# update current iterable and finish
v = self.iterables[i].next()
self.replace_map[self.labels[i]] = v
break
except StopIteration:
# reset the current iterable and set to first element
self.iterables[i] = self.mod.cfields[i][1].__iter__()
v = self.iterables[i].next()
self.replace_map[self.labels[i]] = v
else:
# last iterable has finished, have generated full set
raise StopIteration
return None
def write_header(self, fout):
fout.write("# format:\n# <config index>")
for l in self.labels:
fout.write(" <" + l + ">")
fout.write("\n")
def write_config_line(self, ident, fout):
# print the configuration to the log
fout.write(str(ident))
for l in self.labels:
fout.write(' ' + str(self.replace_map[l]))
else:
fout.write('\n')
# checks - make sure cfields is set and is the correct type
def check_cfields(module):
if "cfields" not in module.__dict__:
raise TypeError("Expected cfields to be defined in " + str(module))
elif not \
(isinstance(module.cfields, Sequence) and \
isinstance(module.cfields[0][0], str) and \
isinstance(module.cfields[0][1], Sequence)):
raise TypeError("cfields in incorrect format, see usage")
# import a python file (assumes there is a .py suffix!!!)
def import_from(filename):
path,name = os.path.split(filename)
name,ext = os.path.splitext(name)
fmod, fname, data = imp.find_module(name, [path])
return imp.load_module(name,fmod,fname,data)
# attempts casting to a numeric type, returning None on failure
def try_num(s):
try:
return int(s)
except ValueError:
try:
return float(s)
except ValueError:
return None
# given a string and a set of substitution pairs,
# perform a single pass replace
# st is the string to perform substitution on
# kv_pairs is a dict or a sequence of sequences.
# kv_pairs examples:
# { a:b, c:d }
# [[a,b],[c,d]]
# [(a,b),(c,d)]
# ((a,b),(c,d))
def replace_many(st, kv_pairs):
rep_dict = {}
# dict-ify and string-ify the input
if isinstance(kv_pairs, dict):
for k in kv_pairs:
rep_dict[k] = str(kv_pairs[k])
elif isinstance(kv_pairs, Sequence) and isinstance(kv_pairs[0], Sequence):
for k in kv_pairs:
rep_dict[k[0]] = str(kv_pairs[k[1]])
else:
raise TypeError("Expected dict or sequence of sequences types")
pat = re.compile("|".join([re.escape(k) for k in rep_dict]))
return pat.sub(lambda match: rep_dict[match.group(0)], st)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment