Commit 7c27e1e3 authored by Paul Rich's avatar Paul Rich

fix for a display issue with cluster node names that have additional device information.

parent 7f42b786
......@@ -275,8 +275,16 @@ def dgetopt(arglist, opt, vopt, msg):
def merge_nodelist(locations):
'''create a set of dashed-ranges from a node list
location - a list of string location identifiers for a system
A string system-appropriate compact represetntation of the location
reg = re.compile(r'(\D+)(\d+)')
device_reg = re.compile(r'(\D+\d+-\D+)(\d)')
cray_loc = re.compile(r'^(\d+)')
# create a dictionary with a key for each node prefix,
# with sorted lists of node numbers as the values
......@@ -284,8 +292,16 @@ def merge_nodelist(locations):
prefix_min_digits = {}
prefix_max_digits = {}
prefix_format_str = {}
has_device_naming = False
ret = []
if (len(locations) > 1 and
reg.match(locations[0]) is None and
device_reg.match(locations[0]) is None and
cray_loc.match(locations[0]) is None):
# We don't know how to merge this list of locations.
return ','.join(sorted(locations))
#if this doesn't have a prefix, like a Cray nidlist, short circut
if len(locations) >= 1:
if cray_loc.match(locations[0]) is not None:
......@@ -296,6 +312,10 @@ def merge_nodelist(locations):
prefix = reg.match(name).group(1)
nodenum = reg.match(name).group(2)
if device_reg.match(name):
has_device_naming = True
prefix = device_reg.match(name).group(1)
nodenum = device_reg.match(name).group(2)
num_digits = len(nodenum)
newnum = int(nodenum)
if not prefix in noderanges:
......@@ -325,6 +345,62 @@ def merge_nodelist(locations):
prefix_format_str[prefix] = {'compact': "[%s%s-%s]",
'single': "%s%s"}
ret = _gen_compacted_list(noderanges, prefix_format_str)
# Compact Nodes and Devices
if has_device_naming:
ret = _compact_device_list(ret)
return ','.join(sorted(ret, cmp=_compare_on_low_range))
def _compact_device_list(locations):
'''compact down node ids for nodes with the same device alllocations.
this is a second stage for the more complex device-split locations
ret = []
node_and_dev = re.compile(r'(\D+\d+)-(\D+[\[\]\-\d]+)')
nodes_by_dev = {}
for node in sorted(locations):
match = node_and_dev.match(node)
if match is None:
node_name =
dev_set =
if nodes_by_dev.get(dev_set, None):
nodes_by_dev[dev_set] = [node_name]
for dev_set, nodes in nodes_by_dev.items():
compacted_nodes = merge_nodelist(nodes).split(',')
ret.extend(["%s-%s" % (node, dev_set) for node in compacted_nodes])
return ret
def _compare_on_low_range(left, right):
'''comparison to sort a compressed list'''
first_number_reg = re.compile(r'(\d+)')
left_num =
right_num =
if left_num == right_num:
return 0
elif left_num > right_num:
return 1
return -1
def _gen_compacted_list(noderanges, prefix_format_str):
'''generate a prefixed compact list of a range, or the singleton pattern
based on a list of locations
noderanges - list of numeric integer id numbers
prefix_format_str - the formatting string dict with a 'compact' and a
'single' format
A list of strings of the form name[X-Y] or nameX
ret = []
# iterate through the sorted lists, identifying gaps in the sequential numbers
for prefix in sorted(noderanges.keys()):
......@@ -347,8 +423,7 @@ def merge_nodelist(locations):
ret.append(prefix_format_str[prefix]['single'] %
(prefix, noderanges[prefix][brk[0]]))
return ','.join(sorted(ret))
return ret
def dgetopt_long(arglist, opt, vopt, msg):
'''parse options into a dictionary, long and short options supported'''
......@@ -1279,4 +1354,4 @@ def sanitize_password(message):
def get_current_thread_identifier():
current_thread = threading.current_thread()
return "%s-%s" % (, current_thread.ident)
\ No newline at end of file
return "%s-%s" % (, current_thread.ident)
......@@ -20,6 +20,9 @@ NoOptionError = Cobalt.Util.NoOptionError
import Cobalt.Exceptions
TimerException = Cobalt.Exceptions.TimerException
from testsuite.TestCobalt.Utilities.assert_functions import assert_match
class TestTimers (object):
def test_elapsed_timer(self, reps = 5):
sleep_time = 1
......@@ -379,3 +382,81 @@ setting = 2
class TestMergeNodelist(object):
'''Tests for Cobalt.Util.merge_nodelist used on cluster systems'''
def test_merge_nodelist_cray_style(self):
'''Cray style nid-list merging'''
correct = '1,3,5-7,112'
resp = Cobalt.Util.merge_nodelist(["112", "1", "3", "5", "6", "7"])
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cray_style_single(self):
'''Cray style nid-list single entry'''
correct = '42'
resp = Cobalt.Util.merge_nodelist(["42"])
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cluster_single(self):
'''cluster system style single entry'''
correct = 'cc42'
resp = Cobalt.Util.merge_nodelist(["cc42.test"])
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cluster(self):
'''cluster system style full nodes'''
correct = 'cc01,cc[42-44]'
resp = Cobalt.Util.merge_nodelist(["cc43.test", "cc42.test",
"cc01.test", "cc44.test"])
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cluster_gpu(self):
'''cluster system style gpu-naming from single node'''
correct = 'cc01-gpu[0-3]'
resp = Cobalt.Util.merge_nodelist(["cc01-gpu2.test", "cc01-gpu0.test",
"cc01-gpu1.test", "cc01-gpu3.test"])
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cluster_gpu_multi_top_nodes(self):
'''cluster system style gpu-naming from multiple nodes'''
correct = 'cc01-gpu[2-3],cc02-gpu[0-1]'
resp = Cobalt.Util.merge_nodelist(["cc01-gpu2.test", "cc02-gpu0.test",
"cc02-gpu1.test", "cc01-gpu3.test"])
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cluster_gpu_split_gpu_multi_host(self):
'''cluster system style gpu-naming split up gpus'''
correct = 'cc[01-03]-gpu[0-3],cc04-gpu[4-7],cc05-gpu[0-3],cc06-gpu0,cc[07-08]-gpu5'
location = ['cc01-gpu0', 'cc01-gpu1', 'cc01-gpu2', 'cc01-gpu3',
'cc02-gpu0', 'cc02-gpu1', 'cc02-gpu2', 'cc02-gpu3',
'cc05-gpu0', 'cc05-gpu1', 'cc05-gpu2', 'cc05-gpu3',
'cc03-gpu0', 'cc03-gpu1', 'cc03-gpu2', 'cc03-gpu3',
'cc04-gpu4', 'cc04-gpu5', 'cc04-gpu6', 'cc04-gpu7',
'cc06-gpu0', 'cc07-gpu5', 'cc08-gpu5',
resp = Cobalt.Util.merge_nodelist(location)
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_nonconforming(self):
'''pass through location lists that do not conform to one of our regexes'''
correct = 'bar,baz,foo'
location = ['foo', 'bar', 'baz']
resp = Cobalt.Util.merge_nodelist(location)
assert_match(resp, correct, "Incorrect merged list")
def test_merge_nodelist_cluster_gpu_split_mixed(self):
'''cluster system style gpu-naming mixed with standard'''
correct = 'cc01,cc[01-03]-gpu[0-3],cc04-gpu[4-7],cc05-gpu[0-3],cc06-gpu0,cc07,cc[08-09]-gpu5,cc[10-12],cc42'
location = ['cc01-gpu0', 'cc01-gpu1', 'cc01-gpu2', 'cc01-gpu3',
'cc02-gpu0', 'cc02-gpu1', 'cc02-gpu2', 'cc02-gpu3',
'cc10', 'cc11', 'cc12', 'cc42',
'cc05-gpu0', 'cc05-gpu1', 'cc05-gpu2', 'cc05-gpu3',
'cc03-gpu0', 'cc03-gpu1', 'cc03-gpu2', 'cc03-gpu3',
'cc04-gpu4', 'cc04-gpu5', 'cc04-gpu6', 'cc04-gpu7',
'cc06-gpu0', 'cc09-gpu5', 'cc08-gpu5', 'cc01', 'cc07',
resp = Cobalt.Util.merge_nodelist(location)
assert_match(resp, correct, "Incorrect merged list")
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment