Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
C
codes
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
38
Issues
38
List
Boards
Labels
Milestones
Merge Requests
8
Merge Requests
8
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
codes
codes
Commits
da5b6b71
Commit
da5b6b71
authored
Mar 27, 2014
by
Jonathan Jenkins
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
filtering of auto-generated config files by fields
parent
3c968fb1
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
209 additions
and
117 deletions
+209
-117
scripts/codes_configurator.py
scripts/codes_configurator.py
+19
-117
scripts/codes_filter_configs.py
scripts/codes_filter_configs.py
+59
-0
scripts/configurator.py
scripts/configurator.py
+131
-0
No files found.
scripts/codes_configurator.py
View file @
da5b6b71
#!/usr/bin/python2.7
import
re
import
sys
import
os
import
imp
import
argparse
from
collections
import
Sequence
import
configurator
as
conf
def
main
():
args
=
parse_args
()
...
...
@@ -14,123 +11,38 @@ def main():
tstr
=
open
(
args
.
template
).
read
()
# load the module
mod
=
import_from
(
args
.
substitute_py
)
mod
=
conf
.
import_from
(
args
.
substitute_py
)
# checks - make sure cfields is set and is the correct type
if
"cfields"
not
in
mod
.
__dict__
:
raise
TypeError
(
"Expected cfields to be defined in "
+
args
.
substitute_py
)
elif
not
\
(
isinstance
(
mod
.
cfields
,
Sequence
)
and
\
isinstance
(
mod
.
cfields
[
0
][
0
],
str
)
and
\
isinstance
(
mod
.
cfields
[
0
][
1
],
Sequence
)):
raise
TypeError
(
"cfields in incorrect format, see usage"
)
num_fields
=
len
(
mod
.
cfields
)
# get list of iterators, replacement input, and output labels
labels
=
[
k
[
0
]
for
k
in
mod
.
cfields
]
iterables
=
[
k
[
1
].
__iter__
()
for
k
in
mod
.
cfields
]
replace_map
=
{
k
[
0
]
:
None
for
k
in
mod
.
cfields
}
conf
.
check_cfields
(
mod
)
# process the command line token replacements, adding to labels
# and replace_map, but not to iterables
# check that pairs are actually pairs
if
len
(
args
.
token_pairs
)
%
2
!=
0
:
raise
ValueError
(
"token pairs must come in twos"
)
elif
len
(
args
.
token_pairs
)
>
0
:
for
i
in
range
(
0
,
len
(
args
.
token_pairs
),
2
):
k
,
vstr
=
args
.
token_pairs
[
i
],
args
.
token_pairs
[
i
+
1
]
# try making v numeric - fall back to string
v
=
try_num
(
vstr
)
if
v
==
None
:
v
=
vstr
# add pair to replace map and labels (but not iterables!)
if
k
in
replace_map
:
raise
ValueError
(
"token "
+
k
+
" of token_pairs matches a token "
"given by the substitution py file"
)
else
:
replace_map
[
k
]
=
v
labels
.
append
(
k
)
# intitialize the configuration object
cfg
=
conf
.
configurator
(
mod
,
args
.
token_pairs
)
# print the header to the log
if
args
.
log
!=
None
:
flog
=
open
(
args
.
log
,
"w"
)
else
:
flog
=
open
(
os
.
devnull
,
"w"
)
flog
.
write
(
"# format:
\
n
# <config index>"
)
for
l
in
labels
:
flog
.
write
(
" <"
+
l
+
">"
)
flog
.
write
(
"
\
n
"
)
# generate initial configuration
for
i
in
range
(
0
,
num_fields
):
v
=
iterables
[
i
].
next
()
replace_map
[
labels
[
i
]]
=
v
cfg
.
write_header
(
flog
)
# main loop
ct
=
0
while
True
:
# generate new configuration
new_config
=
replace_many
(
tstr
,
replace_map
)
fname
=
template
+
"."
+
str
(
ct
)
if
args
.
output_prefix
==
None
else
\
args
.
output_prefix
+
"."
+
str
(
ct
)
# main loop (cfg iterator returns nothing, eval'd for side effects)
for
i
,
_
in
enumerate
(
cfg
):
new_config
=
conf
.
replace_many
(
tstr
,
cfg
.
replace_map
)
fname
=
template
+
"."
+
str
(
i
)
if
args
.
output_prefix
==
None
else
\
args
.
output_prefix
+
"."
+
str
(
i
)
with
open
(
fname
,
"w"
)
as
fout
:
fout
.
write
(
new_config
)
# write this config to the log
cfg
.
write_config_line
(
i
,
flog
)
# print the configuration to the log
flog
.
write
(
str
(
ct
))
for
l
in
labels
:
flog
.
write
(
' '
+
str
(
replace_map
[
l
]))
else
:
flog
.
write
(
'
\
n
'
)
# generate the next config
for
i
in
range
(
0
,
num_fields
):
try
:
# update current iterable and finish
v
=
iterables
[
i
].
next
()
replace_map
[
labels
[
i
]]
=
v
ct
+=
1
break
except
StopIteration
:
# reset the current iterable and set to first element
iterables
[
i
]
=
mod
.
cfields
[
i
][
1
].
__iter__
()
v
=
iterables
[
i
].
next
()
replace_map
[
labels
[
i
]]
=
v
else
:
# last iterable has finished, have generated full set
break
# import a python file (assumes there is a .py suffix!!!)
def
import_from
(
filename
):
path
,
name
=
os
.
path
.
split
(
filename
)
name
,
ext
=
os
.
path
.
splitext
(
name
)
fmod
,
fname
,
data
=
imp
.
find_module
(
name
,
[
path
])
return
imp
.
load_module
(
name
,
fmod
,
fname
,
data
)
# given a string and a set of substitution pairs,
# perform a single pass replace
# st is the string to perform substitution on
# kv_pairs is a dict or a sequence of sequences.
# kv_pairs examples:
# { a:b, c:d }
# [[a,b],[c,d]]
# [(a,b),(c,d)]
# ((a,b),(c,d))
def
replace_many
(
st
,
kv_pairs
):
rep_dict
=
{}
# dict-ify and string-ify the input
if
isinstance
(
kv_pairs
,
dict
):
for
k
in
kv_pairs
:
rep_dict
[
k
]
=
str
(
kv_pairs
[
k
])
elif
isinstance
(
kv_pairs
,
Sequence
)
and
isinstance
(
kv_pairs
[
0
],
Sequence
):
for
k
in
kv_pairs
:
rep_dict
[
k
[
0
]]
=
str
(
kv_pairs
[
k
[
1
]])
else
:
raise
TypeError
(
"Expected dict or sequence of sequences types"
)
pat
=
re
.
compile
(
"|"
.
join
([
re
.
escape
(
k
)
for
k
in
rep_dict
]))
return
pat
.
sub
(
lambda
match
:
rep_dict
[
match
.
group
(
0
)],
st
)
flog
.
close
()
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
\
...
...
@@ -142,8 +54,7 @@ def parse_args():
help
=
'python file defining "cfields" variable consisting of '
'elements of the form '
'( replacement_token, [replacements...])'
)
parser
.
add_argument
(
"token_pairs"
,
nargs
=
'*'
,
parser
.
add_argument
(
"token_pairs"
,
nargs
=
'*'
,
help
=
"a list of whitespace-separated token, replace pairs for "
"command-line-driven replacement (useful in shell scripts "
"for generating sets of configuration files with a distinct "
...
...
@@ -155,14 +66,5 @@ def parse_args():
"(default: the configuration index appended to the template name)"
)
return
parser
.
parse_args
()
def
try_num
(
s
):
try
:
return
int
(
s
)
except
ValueError
:
try
:
return
float
(
s
)
except
ValueError
:
return
None
if
__name__
==
"__main__"
:
main
()
scripts/codes_filter_configs.py
0 → 100755
View file @
da5b6b71
#!/usr/bin/python2.7
import
configurator
as
conf
from
sys
import
stdout
import
argparse
def
main
():
args
=
parse_args
()
# load and check the module
mod
=
conf
.
import_from
(
args
.
substitute_py
)
conf
.
check_cfields
(
mod
)
# check that pairs are actually pairs
tp
=
args
.
token_pairs
if
len
(
tp
)
%
2
!=
0
:
raise
ValueError
(
"token pairs must come in twos"
)
# intitialize the configuration object
cfg
=
conf
.
configurator
(
mod
,
[])
# build a lookup structure for the pairs, converting to numbers if
# possible
tdict
=
{
}
for
i
in
range
(
0
,
len
(
tp
),
2
):
v
=
conf
.
try_num
(
tp
[
i
+
1
])
if
v
==
None
:
v
=
tp
[
i
+
1
]
tdict
[
tp
[
i
]]
=
v
# quick check - make sure pairs are actually in the config
for
k
in
tdict
:
if
k
not
in
cfg
.
replace_map
:
raise
ValueError
(
"key "
+
k
+
" not in original config"
)
# filter out the files
# (cfg iterator returns nothing, eval'd for side effects)
for
i
,
_
in
enumerate
(
cfg
):
for
k
in
tdict
:
if
tdict
[
k
]
!=
cfg
.
replace_map
[
k
]:
break
else
:
stdout
.
write
(
args
.
output_prefix
+
"."
+
str
(
i
)
+
"
\
n
"
)
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
\
description
=
"emit configuration files to stdout matching given "
"token values"
)
parser
.
add_argument
(
"output_prefix"
,
help
=
"prefix of output config files"
)
parser
.
add_argument
(
"substitute_py"
,
help
=
"input python file given to codes_configurator.py - see "
"codes_configurator.py for details"
)
parser
.
add_argument
(
"token_pairs"
,
nargs
=
'*'
,
help
=
"a list of whitespace-separated token, replace pairs to "
"filter the set of generated configuration files by"
)
return
parser
.
parse_args
()
if
__name__
==
"__main__"
:
main
()
scripts/configurator.py
0 → 100644
View file @
da5b6b71
""" Helpers and utilities for use by codes_configurator and friends """
import
re
import
os
import
imp
from
collections
import
Sequence
class
configurator
:
# note: object assumes input has been validated by caller, though it will
# still throw if it finds duplicate keys in the dict
def
__init__
(
self
,
module
,
replace_pairs
):
self
.
mod
=
module
self
.
num_fields
=
len
(
self
.
mod
.
cfields
)
self
.
labels
=
[
k
[
0
]
for
k
in
self
.
mod
.
cfields
]
+
replace_pairs
[
0
::
2
]
self
.
replace_map
=
{
k
[
0
]
:
None
for
k
in
self
.
mod
.
cfields
}
self
.
start_iter
=
False
self
.
in_iter
=
False
for
i
in
range
(
0
,
len
(
replace_pairs
),
2
):
k
,
vstr
=
replace_pairs
[
i
],
replace_pairs
[
i
+
1
]
# add pair to replace map and labels (but not iterables!)
if
k
in
self
.
replace_map
:
raise
ValueError
(
"token "
+
k
+
" of token_pairs matches a token "
"given by the substitution py file"
)
# try making v numeric - fall back to string
v
=
try_num
(
vstr
)
if
v
==
None
:
v
=
vstr
self
.
replace_map
[
k
]
=
v
def
__iter__
(
self
):
# pre-generate an initial config and return self
self
.
start_iter
=
True
self
.
in_iter
=
True
return
self
def
next
(
self
):
if
not
self
.
in_iter
:
# either uninitialized or at the end of the road
raise
StopIteration
elif
self
.
start_iter
:
# first iteration - initialize the iterators
self
.
iterables
=
[
k
[
1
].
__iter__
()
for
k
in
self
.
mod
.
cfields
]
for
i
in
range
(
0
,
self
.
num_fields
):
v
=
self
.
iterables
[
i
].
next
()
self
.
replace_map
[
self
.
labels
[
i
]]
=
v
self
.
start_iter
=
False
else
:
# > first iteration, perform the updates
# generate the next config
for
i
in
range
(
0
,
self
.
num_fields
):
try
:
# update current iterable and finish
v
=
self
.
iterables
[
i
].
next
()
self
.
replace_map
[
self
.
labels
[
i
]]
=
v
break
except
StopIteration
:
# reset the current iterable and set to first element
self
.
iterables
[
i
]
=
self
.
mod
.
cfields
[
i
][
1
].
__iter__
()
v
=
self
.
iterables
[
i
].
next
()
self
.
replace_map
[
self
.
labels
[
i
]]
=
v
else
:
# last iterable has finished, have generated full set
raise
StopIteration
return
None
def
write_header
(
self
,
fout
):
fout
.
write
(
"# format:
\
n
# <config index>"
)
for
l
in
self
.
labels
:
fout
.
write
(
" <"
+
l
+
">"
)
fout
.
write
(
"
\
n
"
)
def
write_config_line
(
self
,
ident
,
fout
):
# print the configuration to the log
fout
.
write
(
str
(
ident
))
for
l
in
self
.
labels
:
fout
.
write
(
' '
+
str
(
self
.
replace_map
[
l
]))
else
:
fout
.
write
(
'
\
n
'
)
# checks - make sure cfields is set and is the correct type
def
check_cfields
(
module
):
if
"cfields"
not
in
module
.
__dict__
:
raise
TypeError
(
"Expected cfields to be defined in "
+
str
(
module
))
elif
not
\
(
isinstance
(
module
.
cfields
,
Sequence
)
and
\
isinstance
(
module
.
cfields
[
0
][
0
],
str
)
and
\
isinstance
(
module
.
cfields
[
0
][
1
],
Sequence
)):
raise
TypeError
(
"cfields in incorrect format, see usage"
)
# import a python file (assumes there is a .py suffix!!!)
def
import_from
(
filename
):
path
,
name
=
os
.
path
.
split
(
filename
)
name
,
ext
=
os
.
path
.
splitext
(
name
)
fmod
,
fname
,
data
=
imp
.
find_module
(
name
,
[
path
])
return
imp
.
load_module
(
name
,
fmod
,
fname
,
data
)
# attempts casting to a numeric type, returning None on failure
def
try_num
(
s
):
try
:
return
int
(
s
)
except
ValueError
:
try
:
return
float
(
s
)
except
ValueError
:
return
None
# given a string and a set of substitution pairs,
# perform a single pass replace
# st is the string to perform substitution on
# kv_pairs is a dict or a sequence of sequences.
# kv_pairs examples:
# { a:b, c:d }
# [[a,b],[c,d]]
# [(a,b),(c,d)]
# ((a,b),(c,d))
def
replace_many
(
st
,
kv_pairs
):
rep_dict
=
{}
# dict-ify and string-ify the input
if
isinstance
(
kv_pairs
,
dict
):
for
k
in
kv_pairs
:
rep_dict
[
k
]
=
str
(
kv_pairs
[
k
])
elif
isinstance
(
kv_pairs
,
Sequence
)
and
isinstance
(
kv_pairs
[
0
],
Sequence
):
for
k
in
kv_pairs
:
rep_dict
[
k
[
0
]]
=
str
(
kv_pairs
[
k
[
1
]])
else
:
raise
TypeError
(
"Expected dict or sequence of sequences types"
)
pat
=
re
.
compile
(
"|"
.
join
([
re
.
escape
(
k
)
for
k
in
rep_dict
]))
return
pat
.
sub
(
lambda
match
:
rep_dict
[
match
.
group
(
0
)],
st
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment