Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Xin Wang
codes-dev
Commits
da5b6b71
Commit
da5b6b71
authored
Mar 27, 2014
by
Jonathan Jenkins
Browse files
filtering of auto-generated config files by fields
parent
3c968fb1
Changes
3
Hide whitespace changes
Inline
Side-by-side
scripts/codes_configurator.py
View file @
da5b6b71
#!/usr/bin/python2.7
import
re
import
sys
import
os
import
imp
import
argparse
from
collections
import
Sequence
import
configurator
as
conf
def
main
():
args
=
parse_args
()
...
...
@@ -14,123 +11,38 @@ def main():
tstr
=
open
(
args
.
template
).
read
()
# load the module
mod
=
import_from
(
args
.
substitute_py
)
mod
=
conf
.
import_from
(
args
.
substitute_py
)
# checks - make sure cfields is set and is the correct type
if
"cfields"
not
in
mod
.
__dict__
:
raise
TypeError
(
"Expected cfields to be defined in "
+
args
.
substitute_py
)
elif
not
\
(
isinstance
(
mod
.
cfields
,
Sequence
)
and
\
isinstance
(
mod
.
cfields
[
0
][
0
],
str
)
and
\
isinstance
(
mod
.
cfields
[
0
][
1
],
Sequence
)):
raise
TypeError
(
"cfields in incorrect format, see usage"
)
num_fields
=
len
(
mod
.
cfields
)
# get list of iterators, replacement input, and output labels
labels
=
[
k
[
0
]
for
k
in
mod
.
cfields
]
iterables
=
[
k
[
1
].
__iter__
()
for
k
in
mod
.
cfields
]
replace_map
=
{
k
[
0
]
:
None
for
k
in
mod
.
cfields
}
conf
.
check_cfields
(
mod
)
# process the command line token replacements, adding to labels
# and replace_map, but not to iterables
# check that pairs are actually pairs
if
len
(
args
.
token_pairs
)
%
2
!=
0
:
raise
ValueError
(
"token pairs must come in twos"
)
elif
len
(
args
.
token_pairs
)
>
0
:
for
i
in
range
(
0
,
len
(
args
.
token_pairs
),
2
):
k
,
vstr
=
args
.
token_pairs
[
i
],
args
.
token_pairs
[
i
+
1
]
# try making v numeric - fall back to string
v
=
try_num
(
vstr
)
if
v
==
None
:
v
=
vstr
# add pair to replace map and labels (but not iterables!)
if
k
in
replace_map
:
raise
ValueError
(
"token "
+
k
+
" of token_pairs matches a token "
"given by the substitution py file"
)
else
:
replace_map
[
k
]
=
v
labels
.
append
(
k
)
# intitialize the configuration object
cfg
=
conf
.
configurator
(
mod
,
args
.
token_pairs
)
# print the header to the log
if
args
.
log
!=
None
:
flog
=
open
(
args
.
log
,
"w"
)
else
:
flog
=
open
(
os
.
devnull
,
"w"
)
flog
.
write
(
"# format:
\n
# <config index>"
)
for
l
in
labels
:
flog
.
write
(
" <"
+
l
+
">"
)
flog
.
write
(
"
\n
"
)
# generate initial configuration
for
i
in
range
(
0
,
num_fields
):
v
=
iterables
[
i
].
next
()
replace_map
[
labels
[
i
]]
=
v
cfg
.
write_header
(
flog
)
# main loop
ct
=
0
while
True
:
# generate new configuration
new_config
=
replace_many
(
tstr
,
replace_map
)
fname
=
template
+
"."
+
str
(
ct
)
if
args
.
output_prefix
==
None
else
\
args
.
output_prefix
+
"."
+
str
(
ct
)
# main loop (cfg iterator returns nothing, eval'd for side effects)
for
i
,
_
in
enumerate
(
cfg
):
new_config
=
conf
.
replace_many
(
tstr
,
cfg
.
replace_map
)
fname
=
template
+
"."
+
str
(
i
)
if
args
.
output_prefix
==
None
else
\
args
.
output_prefix
+
"."
+
str
(
i
)
with
open
(
fname
,
"w"
)
as
fout
:
fout
.
write
(
new_config
)
# write this config to the log
cfg
.
write_config_line
(
i
,
flog
)
# print the configuration to the log
flog
.
write
(
str
(
ct
))
for
l
in
labels
:
flog
.
write
(
' '
+
str
(
replace_map
[
l
]))
else
:
flog
.
write
(
'
\n
'
)
# generate the next config
for
i
in
range
(
0
,
num_fields
):
try
:
# update current iterable and finish
v
=
iterables
[
i
].
next
()
replace_map
[
labels
[
i
]]
=
v
ct
+=
1
break
except
StopIteration
:
# reset the current iterable and set to first element
iterables
[
i
]
=
mod
.
cfields
[
i
][
1
].
__iter__
()
v
=
iterables
[
i
].
next
()
replace_map
[
labels
[
i
]]
=
v
else
:
# last iterable has finished, have generated full set
break
# import a python file (assumes there is a .py suffix!!!)
def
import_from
(
filename
):
path
,
name
=
os
.
path
.
split
(
filename
)
name
,
ext
=
os
.
path
.
splitext
(
name
)
fmod
,
fname
,
data
=
imp
.
find_module
(
name
,
[
path
])
return
imp
.
load_module
(
name
,
fmod
,
fname
,
data
)
# given a string and a set of substitution pairs,
# perform a single pass replace
# st is the string to perform substitution on
# kv_pairs is a dict or a sequence of sequences.
# kv_pairs examples:
# { a:b, c:d }
# [[a,b],[c,d]]
# [(a,b),(c,d)]
# ((a,b),(c,d))
def
replace_many
(
st
,
kv_pairs
):
rep_dict
=
{}
# dict-ify and string-ify the input
if
isinstance
(
kv_pairs
,
dict
):
for
k
in
kv_pairs
:
rep_dict
[
k
]
=
str
(
kv_pairs
[
k
])
elif
isinstance
(
kv_pairs
,
Sequence
)
and
isinstance
(
kv_pairs
[
0
],
Sequence
):
for
k
in
kv_pairs
:
rep_dict
[
k
[
0
]]
=
str
(
kv_pairs
[
k
[
1
]])
else
:
raise
TypeError
(
"Expected dict or sequence of sequences types"
)
pat
=
re
.
compile
(
"|"
.
join
([
re
.
escape
(
k
)
for
k
in
rep_dict
]))
return
pat
.
sub
(
lambda
match
:
rep_dict
[
match
.
group
(
0
)],
st
)
flog
.
close
()
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
\
...
...
@@ -142,8 +54,7 @@ def parse_args():
help
=
'python file defining "cfields" variable consisting of '
'elements of the form '
'( replacement_token, [replacements...])'
)
parser
.
add_argument
(
"token_pairs"
,
nargs
=
'*'
,
parser
.
add_argument
(
"token_pairs"
,
nargs
=
'*'
,
help
=
"a list of whitespace-separated token, replace pairs for "
"command-line-driven replacement (useful in shell scripts "
"for generating sets of configuration files with a distinct "
...
...
@@ -155,14 +66,5 @@ def parse_args():
"(default: the configuration index appended to the template name)"
)
return
parser
.
parse_args
()
def
try_num
(
s
):
try
:
return
int
(
s
)
except
ValueError
:
try
:
return
float
(
s
)
except
ValueError
:
return
None
if
__name__
==
"__main__"
:
main
()
scripts/codes_filter_configs.py
0 → 100755
View file @
da5b6b71
#!/usr/bin/python2.7
import
configurator
as
conf
from
sys
import
stdout
import
argparse
def
main
():
args
=
parse_args
()
# load and check the module
mod
=
conf
.
import_from
(
args
.
substitute_py
)
conf
.
check_cfields
(
mod
)
# check that pairs are actually pairs
tp
=
args
.
token_pairs
if
len
(
tp
)
%
2
!=
0
:
raise
ValueError
(
"token pairs must come in twos"
)
# intitialize the configuration object
cfg
=
conf
.
configurator
(
mod
,
[])
# build a lookup structure for the pairs, converting to numbers if
# possible
tdict
=
{
}
for
i
in
range
(
0
,
len
(
tp
),
2
):
v
=
conf
.
try_num
(
tp
[
i
+
1
])
if
v
==
None
:
v
=
tp
[
i
+
1
]
tdict
[
tp
[
i
]]
=
v
# quick check - make sure pairs are actually in the config
for
k
in
tdict
:
if
k
not
in
cfg
.
replace_map
:
raise
ValueError
(
"key "
+
k
+
" not in original config"
)
# filter out the files
# (cfg iterator returns nothing, eval'd for side effects)
for
i
,
_
in
enumerate
(
cfg
):
for
k
in
tdict
:
if
tdict
[
k
]
!=
cfg
.
replace_map
[
k
]:
break
else
:
stdout
.
write
(
args
.
output_prefix
+
"."
+
str
(
i
)
+
"
\n
"
)
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
\
description
=
"emit configuration files to stdout matching given "
"token values"
)
parser
.
add_argument
(
"output_prefix"
,
help
=
"prefix of output config files"
)
parser
.
add_argument
(
"substitute_py"
,
help
=
"input python file given to codes_configurator.py - see "
"codes_configurator.py for details"
)
parser
.
add_argument
(
"token_pairs"
,
nargs
=
'*'
,
help
=
"a list of whitespace-separated token, replace pairs to "
"filter the set of generated configuration files by"
)
return
parser
.
parse_args
()
if
__name__
==
"__main__"
:
main
()
scripts/configurator.py
0 → 100644
View file @
da5b6b71
""" Helpers and utilities for use by codes_configurator and friends """
import
re
import
os
import
imp
from
collections
import
Sequence
class
configurator
:
# note: object assumes input has been validated by caller, though it will
# still throw if it finds duplicate keys in the dict
def
__init__
(
self
,
module
,
replace_pairs
):
self
.
mod
=
module
self
.
num_fields
=
len
(
self
.
mod
.
cfields
)
self
.
labels
=
[
k
[
0
]
for
k
in
self
.
mod
.
cfields
]
+
replace_pairs
[
0
::
2
]
self
.
replace_map
=
{
k
[
0
]
:
None
for
k
in
self
.
mod
.
cfields
}
self
.
start_iter
=
False
self
.
in_iter
=
False
for
i
in
range
(
0
,
len
(
replace_pairs
),
2
):
k
,
vstr
=
replace_pairs
[
i
],
replace_pairs
[
i
+
1
]
# add pair to replace map and labels (but not iterables!)
if
k
in
self
.
replace_map
:
raise
ValueError
(
"token "
+
k
+
" of token_pairs matches a token "
"given by the substitution py file"
)
# try making v numeric - fall back to string
v
=
try_num
(
vstr
)
if
v
==
None
:
v
=
vstr
self
.
replace_map
[
k
]
=
v
def
__iter__
(
self
):
# pre-generate an initial config and return self
self
.
start_iter
=
True
self
.
in_iter
=
True
return
self
def
next
(
self
):
if
not
self
.
in_iter
:
# either uninitialized or at the end of the road
raise
StopIteration
elif
self
.
start_iter
:
# first iteration - initialize the iterators
self
.
iterables
=
[
k
[
1
].
__iter__
()
for
k
in
self
.
mod
.
cfields
]
for
i
in
range
(
0
,
self
.
num_fields
):
v
=
self
.
iterables
[
i
].
next
()
self
.
replace_map
[
self
.
labels
[
i
]]
=
v
self
.
start_iter
=
False
else
:
# > first iteration, perform the updates
# generate the next config
for
i
in
range
(
0
,
self
.
num_fields
):
try
:
# update current iterable and finish
v
=
self
.
iterables
[
i
].
next
()
self
.
replace_map
[
self
.
labels
[
i
]]
=
v
break
except
StopIteration
:
# reset the current iterable and set to first element
self
.
iterables
[
i
]
=
self
.
mod
.
cfields
[
i
][
1
].
__iter__
()
v
=
self
.
iterables
[
i
].
next
()
self
.
replace_map
[
self
.
labels
[
i
]]
=
v
else
:
# last iterable has finished, have generated full set
raise
StopIteration
return
None
def
write_header
(
self
,
fout
):
fout
.
write
(
"# format:
\n
# <config index>"
)
for
l
in
self
.
labels
:
fout
.
write
(
" <"
+
l
+
">"
)
fout
.
write
(
"
\n
"
)
def
write_config_line
(
self
,
ident
,
fout
):
# print the configuration to the log
fout
.
write
(
str
(
ident
))
for
l
in
self
.
labels
:
fout
.
write
(
' '
+
str
(
self
.
replace_map
[
l
]))
else
:
fout
.
write
(
'
\n
'
)
# checks - make sure cfields is set and is the correct type
def
check_cfields
(
module
):
if
"cfields"
not
in
module
.
__dict__
:
raise
TypeError
(
"Expected cfields to be defined in "
+
str
(
module
))
elif
not
\
(
isinstance
(
module
.
cfields
,
Sequence
)
and
\
isinstance
(
module
.
cfields
[
0
][
0
],
str
)
and
\
isinstance
(
module
.
cfields
[
0
][
1
],
Sequence
)):
raise
TypeError
(
"cfields in incorrect format, see usage"
)
# import a python file (assumes there is a .py suffix!!!)
def
import_from
(
filename
):
path
,
name
=
os
.
path
.
split
(
filename
)
name
,
ext
=
os
.
path
.
splitext
(
name
)
fmod
,
fname
,
data
=
imp
.
find_module
(
name
,
[
path
])
return
imp
.
load_module
(
name
,
fmod
,
fname
,
data
)
# attempts casting to a numeric type, returning None on failure
def
try_num
(
s
):
try
:
return
int
(
s
)
except
ValueError
:
try
:
return
float
(
s
)
except
ValueError
:
return
None
# given a string and a set of substitution pairs,
# perform a single pass replace
# st is the string to perform substitution on
# kv_pairs is a dict or a sequence of sequences.
# kv_pairs examples:
# { a:b, c:d }
# [[a,b],[c,d]]
# [(a,b),(c,d)]
# ((a,b),(c,d))
def
replace_many
(
st
,
kv_pairs
):
rep_dict
=
{}
# dict-ify and string-ify the input
if
isinstance
(
kv_pairs
,
dict
):
for
k
in
kv_pairs
:
rep_dict
[
k
]
=
str
(
kv_pairs
[
k
])
elif
isinstance
(
kv_pairs
,
Sequence
)
and
isinstance
(
kv_pairs
[
0
],
Sequence
):
for
k
in
kv_pairs
:
rep_dict
[
k
[
0
]]
=
str
(
kv_pairs
[
k
[
1
]])
else
:
raise
TypeError
(
"Expected dict or sequence of sequences types"
)
pat
=
re
.
compile
(
"|"
.
join
([
re
.
escape
(
k
)
for
k
in
rep_dict
]))
return
pat
.
sub
(
lambda
match
:
rep_dict
[
match
.
group
(
0
)],
st
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment