listgen-upd.py 7.47 KB
Newer Older
1 2 3 4
import sys
import random
alloc_file = 'testrest.conf'

5
def contiguous_alloc(job_ranks, total_nodes, cores_per_node):
6 7 8 9 10 11 12 13 14
    f = open(alloc_file,'w')
    start=0
    for num_rank in range(len(job_ranks)):
        for rankid in range(start, start+job_ranks[num_rank]):
            f.write("%s " % rankid)
        f.write("\n")
        start += job_ranks[num_rank]
    f.closed

15
def cube_alloc(job_ranks, total_nodes, cores_per_node):
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
    job_dim = [6,6,6]
    sys_dim_x = 16
    sys_dim_y =16
    sys_dim_z = 8
    cube = []
    start = 0
    for k in range(job_dim[2]):
        layer = []
        layer_offset = k*sys_dim_x*sys_dim_y
        for j in range(job_dim[1]):
            row_offset = j*sys_dim_z
            row = []
            for i in range(job_dim[0]):
                offset = row_offset+layer_offset
                row.append(i+offset)
            layer += row
        cube += layer
    print "list length is", len(cube), cube

    f = open('cube_allc_linear.conf','w')
    for rankid in range(len(cube)):
        f.write("%s " % cube[rankid])
    f.write("\n")
    f.closed

    f = open('cube_allc_random.conf','w')
    random.shuffle(cube)
    for rankid in range(len(cube)):
        f.write("%s " % cube[rankid])
    f.write("\n")
    f.closed



50
def permeate_alloc(job_ranks, total_nodes, cores_per_node):
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
    f = open(alloc_file,'w')
    start=0
    node_list = range(0, int(total_nodes))
    random.seed(0)
    for num_rank in range(len(job_ranks)):
        permeate_area = job_ranks[num_rank]*4
        permeate_list = node_list[num_rank*permeate_area: (num_rank+1)*permeate_area]
        alloc_list = random.sample(permeate_list, job_ranks[num_rank])
        alloc_list.sort()
        print "length of alloc list", len(alloc_list), "\n", alloc_list,"\n"
        for idx in range(len(alloc_list)):
            f.write("%s " % alloc_list[idx])
        f.write("\n")
    f.closed

66
def random_alloc(job_rank, total_nodes, num_seed, cores_per_node):
67
    filename_substr='rand_node1-alloc-'+str(total_nodes)+'-'
68 69 70 71 72 73 74 75 76 77
    for jobsize in job_rank:
        filename_substr += str(jobsize)+'_'
        #print filename_substr
    filename_substr=filename_substr[:-1]
    for seed in range(num_seed):
        filename_substr += '-'+str(seed)
        #print filename_substr
        f = open(filename_substr+'.conf', 'w')
        node_list = range(0, int(total_nodes))
        random.seed(seed)
78
        
79
        for rankid in range(len(job_rank)):
80 81 82 83 84
            rem = job_rank[rankid]/cores_per_node
            if(job_rank[rankid] % cores_per_node):
                rem=rem+1

            alloc_list = random.sample(node_list, rem)
85 86
            node_list = [i for i in node_list if (i not in alloc_list)]
            #print "length of alloc list", len(alloc_list), "\n", alloc_list,"\n"
87
            count = 0
88
            for idx in range(len(alloc_list)):
89 90 91 92 93 94
                for coreid in range(cores_per_node):
                    if(count == job_rank[rankid]):
                        break
                    f.write("%s " % ((alloc_list[idx] * cores_per_node) + coreid))
                    count=count+1

95 96 97 98
            f.write("\n")
        f.closed
        filename_substr=filename_substr[:-2]

99
def hybrid_alloc(job_rank, total_nodes, cores_per_node):
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
    #1st job get contiguous allocation , the other job get random allocation
    f = open(alloc_file, 'w')
    node_list = range(0, int(total_nodes))
    random.seed(0)
    group_size = 32
    for rankid in range(len(job_rank)):
        if(rankid == 0):
            job_size = job_rank[rankid]
            num_groups = job_size/group_size +1;#job_0 will take this number of groups
            alloc_list = node_list[0: job_rank[rankid]]
            node_list = node_list[num_groups*group_size : ]
        else:
            alloc_list = random.sample(node_list, job_rank[rankid])

        node_list = [i for i in node_list if (i not in alloc_list)]
        print "length of alloc list", len(alloc_list), "\n", alloc_list,"\n"
        for idx in range(len(alloc_list)):
            f.write("%s " % alloc_list[idx])
        f.write("\n")
    f.closed

121
def hybrid_alloc_2 (job_rank, total_nodes, cores_per_node):
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
    #1st and 2nd job get contiguous allocation , 3rd job get random allocation
    f = open(alloc_file, 'w')
    node_list = range(0, int(total_nodes))
    random.seed(0)
    group_size = 32
    for rankid in range(len(job_rank)):
        #if(rankid !=2 2 ):
        if(rankid < 3 ):#all job get contiguous allocation
            job_size = job_rank[rankid]
            num_groups = job_size/group_size +1;#job_0 and job_1 will take this number of groups
            alloc_list = node_list[0: job_rank[rankid]]
            node_list = node_list[num_groups*group_size : ]
        else:
            alloc_list = random.sample(node_list, job_rank[rankid])

        node_list = [i for i in node_list if (i not in alloc_list)]
        print "length of alloc list", len(alloc_list), "\n", alloc_list,"\n"
        for idx in range(len(alloc_list)):
            f.write("%s " % alloc_list[idx])
        f.write("\n")
    f.closed



146
def stripe_alloc(job_ranks, total_nodes, cores_per_node):
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
    #print "the num of nodes of each Job", job_ranks
    f = open(alloc_file,'w')
    node_list = range(0, int(total_nodes))
    stripe_size = 2
    alloc_list = []
    for num_rank in range(len(job_ranks)):
    #    print "job id", num_rank
        num_stripe = 1
        start = num_rank*stripe_size
        if(job_ranks[num_rank] % stripe_size != 0):
            num_stripe = job_ranks[num_rank]/stripe_size+1
        else:
            num_stripe = job_ranks[num_rank]/stripe_size
        tmp_list = []
        while(num_stripe>0):
            tmp_list += node_list[start:start+stripe_size]
            start += len(job_ranks)*stripe_size
            num_stripe -= 1
        alloc_list.append(tmp_list)


    for job_id in range (len(alloc_list)):
        tmp = alloc_list[job_id]
        #print "alloc list for JOB", job_id
        for rankid in range (job_ranks[job_id]):
           # print tmp[rankid]
            f.write("%s " % tmp[rankid])
        f.write("\n")
    f.closed

177
def policy_select(plcy, job_ranks, total_nodes, num_seed, cores_per_node):
178 179
    if plcy == "CONT":
        print "contiguous alloction!"
180
        contiguous_alloc(job_ranks,  total_nodes, cores_per_node)
181 182
    elif plcy == "rand":
        print "random allocation!"
183
        random_alloc(job_ranks, total_nodes, num_seed, cores_per_node)
184 185
    elif plcy == "STRIPE":
        print "stripe allcation!"
186
        stripe_alloc(job_ranks, total_nodes, cores_per_node)
187 188
    elif plcy == "PERMEATE":
        print "permeate allocation!"
189
        permeate_alloc(job_ranks, total_nodes, cores_per_node)
190 191
    elif plcy == "CUBE":
        print "cube allocation!"
192
        cube_alloc(job_ranks, total_nodes, cores_per_node)
193 194
    elif plcy == "hybrid":
        print "hybrid allocation!"
195
        hybrid_alloc(job_ranks, total_nodes, cores_per_node)
196 197
    elif plcy == "hybrid-2":
        print "hybrid 2 allocation!"
198
        hybrid_alloc_2(job_ranks, total_nodes, cores_per_node)
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
    else:
        print "NOT Supported yet!"


if __name__ == "__main__":
    f = open(sys.argv[1], "r")
    array = []

    for line in f:
        for number in line.split():
            array.append(number);

    f.close()
    alloc_plcy = array.pop(0)
    total_nodes = array.pop(0)
    num_seed = int(array.pop(0))
215
    cores_per_node = int(array.pop(0))
216
    print alloc_plcy
217
    print cores_per_node
218 219 220
    array = map(int, array)
    #print array
    #print num_seed, type(num_seed)
221
    policy_select(alloc_plcy, array, total_nodes, num_seed, cores_per_node)
222