codes_mapping.c 13.3 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5
6
7
8
9
10
11
 */

/* SUMMARY:
 * CODES custom mapping file for ROSS
 */
#include "codes/codes_mapping.h"

12
13
#define CODES_MAPPING_DEBUG 0

14
15
16
17
18
19
/* number of LPs assigned to the current PE (abstraction of MPI rank).
 * for lp counts which are not divisible by the number of ranks, keep 
 * modulus around */
static int lps_per_pe_floor = 0;
static int lps_leftover = 0;

Jonathan Jenkins's avatar
Jonathan Jenkins committed
20
static int mem_factor = 256;
21
22
23

static inline int mini(int a, int b){ return a < b ? a : b; }

24
25
26
27
28
/* char arrays for holding lp type name and group name*/
char local_grp_name[MAX_NAME_LENGTH], local_lp_name[MAX_NAME_LENGTH];

config_lpgroups_t lpconf;

29
30
int codes_mapping_get_lps_for_pe()
{
31
32
33
34
35
36
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#if CODES_MAPPING_DEBUG
    printf("%d lps for rank %d\n", lps_per_pe_floor+(g_tw_mynode < lps_leftover), rank);
#endif
  return lps_per_pe_floor + (g_tw_mynode < lps_leftover);
37
38
39
40
41
}

/* Takes the global LP ID and returns the rank (PE id) on which the LP is mapped */
tw_peid codes_mapping( tw_lpid gid)
{
42
43
44
45
46
47
48
49
    int lps_on_pes_with_leftover = lps_leftover * (lps_per_pe_floor+1);
    if (gid < lps_on_pes_with_leftover){
        return gid / (lps_per_pe_floor+1);
    }
    else{
        return (gid-lps_on_pes_with_leftover)/lps_per_pe_floor + lps_leftover;
    }
  /*return gid / lps_per_pe_floor;*/
50
51
}

52
int codes_mapping_get_group_reps(char* grp_name)
53
{
54
55
56
57
58
59
60
61
62
63
  int grp;
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
  {
     if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
	     return lpconf.lpgroups[grp].repetitions;
  }
  return -1;
}
int codes_mapping_get_lp_count(char* grp_name, char* lp_type_name)
{
64
   int grp, lpt, lp_types_count;
65

66
67
// Account for all lps in the previous groups 
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
68
   {
69
70
71
72
73
74
75
76
77
78
       lp_types_count = lpconf.lpgroups[grp].lptypes_count;
        
       if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
	{
	 for(lpt = 0; lpt < lp_types_count; lpt++)
 	 {
	    if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0)
		return lpconf.lpgroups[grp].lptypes[lpt].count;
	 }		 
	}
79
   }
80
  return -1;
81
}
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99

int codes_mapping_get_global_lp_count(char* lp_type_name){
    int total_ct = 0;
    int grp;
    for (grp = 0; grp < lpconf.lpgroups_count; grp++){
        int lpt;
        int lp_types_count = lpconf.lpgroups[grp].lptypes_count;
        int reps = lpconf.lpgroups[grp].repetitions;
        config_lptype_t *c = lpconf.lpgroups[grp].lptypes;
        for (lpt = 0; lpt < lp_types_count; lpt++){
            if(strcmp(c[lpt].name, lp_type_name) == 0){
                total_ct += c[lpt].count * reps;
            }
        }
    }
    return total_ct;
}

100
101
102
103
104
105
106
107
/* This function takes the group ID , type ID and rep ID then returns the global LP ID */
/* TODO: Add string based search for LP group and type names */
void codes_mapping_get_lp_id(char* grp_name, char* lp_type_name, int rep_id, int offset, tw_lpid* gid)
{
 int grp, lpt, lpcount = 0, lp_types_count, rep, count_for_this_lpt;
 short found = 0;

 // Account for all lps in the previous groups 
108
 for(grp = 0; grp < lpconf.lpgroups_count; grp++)
109
  {
110
111
    lp_types_count = lpconf.lpgroups[grp].lptypes_count;
    rep = lpconf.lpgroups[grp].repetitions;
112

113
    if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
114
115
116
117
118
119
    {
	    found = 1;
	    break;
    }
    
    for(lpt = 0; lpt < lp_types_count; lpt++)
120
      lpcount += (rep * lpconf.lpgroups[grp].lptypes[lpt].count);
121
122
123
124
125
  }

 assert(found);
 found = 0;

126
 lp_types_count = lpconf.lpgroups[grp].lptypes_count;
127
128
129
130
 
 // Account for the previous lp types in the current repetition
 for(lpt = 0; lpt < lp_types_count; lpt++)
 {
131
   count_for_this_lpt = lpconf.lpgroups[grp].lptypes[lpt].count;
132

133
   if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0)
134
135
136
137
138
139
140
141
142
143
144
145
146
   {
     found = 1;
     break;
   }
   
   lpcount += count_for_this_lpt;
 }

 assert(found);
 // Account for all previous repetitions
 for(rep = 0; rep < rep_id; rep++)
 {
    for(lpt = 0; lpt < lp_types_count; lpt++)
147
      lpcount += lpconf.lpgroups[grp].lptypes[lpt].count;
148
 }
149
   *gid = lpcount + offset % count_for_this_lpt;
150
151
}

152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
int codes_mapping_get_lp_global_rel_id(tw_lpid gid){
    /* use mapping function to get our starting point */
    char grp_name[MAX_NAME_LENGTH];
    char lp_type_name[MAX_NAME_LENGTH];
    int grp_id, lp_type_id, grp_rep_id, offset;
    codes_mapping_get_lp_info(gid, grp_name, &grp_id, &lp_type_id, lp_type_name,
            &grp_rep_id, &offset);

    /* now, go through the groups that occurred before and count up the lps of
     * the same name */
    int grp;
    int total_lp_count = 0;
    for (grp = 0; grp < grp_id; grp++){
        int rep = lpconf.lpgroups[grp].repetitions;
        int l;
        /* get the total number of LPs in this group */
        for (l = 0; l < lpconf.lpgroups[grp].lptypes_count; l++){
            if (strcmp(lp_type_name, lpconf.lpgroups[grp].lptypes[l].name) == 0){
                total_lp_count += rep * lpconf.lpgroups[grp].lptypes[l].count;
            }
        }
    }

    /* the global relative lp id is my relative position within the group +
     * the number of lps that came before */
    int nlp_per_rep = lpconf.lpgroups[grp_id].lptypes[lp_type_id].count;
    return total_lp_count + (grp_rep_id * nlp_per_rep) + offset;
}


182
183
184
185
186
187
188
189
190
/* This function takes the LP ID and returns its grp index, lp type ID and repetition ID */
void codes_mapping_get_lp_info(tw_lpid gid, char* grp_name, int* grp_id, int* lp_type_id, char* lp_type_name, int* grp_rep_id, int* offset)
{
  int grp, lpt, rep, grp_offset, lp_offset, rep_offset;
  int lp_tmp_id, lp_types_count, lp_count;
  unsigned long grp_lp_count=0;
  short found = 0;
 
  /* Find the group id first */ 
191
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
192
193
194
  {
    grp_offset = 0;
    rep_offset = 0;
195
196
    rep = lpconf.lpgroups[grp].repetitions;
    lp_types_count = lpconf.lpgroups[grp].lptypes_count;
197
198
199

    for(lpt = 0; lpt < lp_types_count; lpt++)
    {
200
	lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
201
202
203
204
205
206
207
208
	grp_offset += (rep * lp_count);
	rep_offset += lp_count;
    }
    /* Each gid is assigned an increasing number starting from 0th group and 0th lp type
     * so we check here if the gid lies within the numeric range of a group */ 
    if(gid >= grp_lp_count && gid < grp_lp_count + grp_offset)
    {
	*grp_id = grp;
209
	 strcpy(local_grp_name, lpconf.lpgroups[grp].name);
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
	 lp_offset = gid - grp_lp_count; /* gets the lp offset starting from the point where the group begins */
         *grp_rep_id = lp_offset / rep_offset;
          lp_tmp_id = lp_offset - (*grp_rep_id * rep_offset);
	  found = 1;
	 break;
    }
    grp_lp_count += grp_offset; /* keep on increasing the group lp count to next group range*/
  }
  assert(found);

  lp_offset = 0;
  found = 0; /* reset found for finding LP type */
 /* Now we compute the LP type ID here based on the lp offset that we just got */ 
  for(lpt = 0; lpt < lp_types_count; lpt++)
  {
225
     lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
226
227
228
     if(lp_tmp_id >= lp_offset && lp_tmp_id < lp_offset + lp_count)
     {
	     *lp_type_id = lpt;
229
	      strcpy(local_lp_name, lpconf.lpgroups[grp].lptypes[lpt].name);
230
231
232
233
234
235
236
237
238
239
240
241
242
	      *offset = lp_tmp_id - lp_offset;
	      found = 1;
     	      break;
     }
     lp_offset += lp_count;
  }
  assert(found);
  strncpy(grp_name, local_grp_name, MAX_NAME_LENGTH);
  strncpy(lp_type_name, local_lp_name, MAX_NAME_LENGTH);
  //printf("\n gid %d lp type name %s rep_id %d ", gid, lp_type_name, *grp_rep_id);
}

/* This function assigns local and global LP Ids to LPs */
243
static void codes_mapping_init(void)
244
245
246
247
248
249
{
     int grp_id, lpt_id, rep_id, offset;
     tw_lpid ross_gid, ross_lid; /* ross global and local IDs */
     tw_pe * pe;
     char lp_type_name[MAX_NAME_LENGTH];
     char grp_name[MAX_NAME_LENGTH];
250
     int nkp_per_pe = g_tw_nkp;
251
252
253
254
255
256
     tw_lpid         lpid, kpid;

     /* have 16 kps per pe, this is the optimized configuration for ROSS custom mapping */
     for(kpid = 0; kpid < nkp_per_pe; kpid++)
	tw_kp_onpe(kpid, g_tw_pe[0]);

257
258
259
260
261
     int lp_start =
         g_tw_mynode * lps_per_pe_floor + mini(g_tw_mynode,lps_leftover);
     int lp_end =
         (g_tw_mynode+1) * lps_per_pe_floor + mini(g_tw_mynode+1,lps_leftover);
     codes_mapping_get_lp_info(lp_start, grp_name, &grp_id, &lpt_id, lp_type_name, &rep_id, &offset);
262

263
     for (lpid = lp_start; lpid < lp_end; lpid++)
264
265
      {
	 ross_gid = lpid;
266
	 ross_lid = lpid - lp_start;
267
268
269
	 kpid = ross_lid % g_tw_nkp;
	 pe = tw_getpe(kpid % g_tw_npe);
	 codes_mapping_get_lp_info(ross_gid, grp_name, &grp_id, &lpt_id, lp_type_name, &rep_id, &offset);
270
271
272
#if CODES_MAPPING_DEBUG
         printf("lp:%lu --> kp:%lu, pe:%llu\n", ross_gid, kpid, pe->id);
#endif
273
	 tw_lp_onpe(ross_lid, pe, ross_gid);
274
275
276
277
278
279
280
281
282
	 tw_lp_onkp(g_tw_lp[ross_lid], g_tw_kp[kpid]);
	 tw_lp_settype(ross_lid, lp_type_lookup(lp_type_name));
     }
     return;
}

/* This function takes the global LP ID, maps it to the local LP ID and returns the LP 
 * lps have global and local LP IDs
 * global LP IDs are unique across all PEs, local LP IDs are unique within a PE */
283
static tw_lp * codes_mapping_to_lp( tw_lpid lpid)
284
{
285
286
   int index = lpid - (g_tw_mynode * lps_per_pe_floor) -
       mini(g_tw_mynode, lps_leftover);
287
288
289
290
//   printf("\n global id %d index %d lps_before %d lps_offset %d local index %d ", lpid, index, lps_before, g_tw_mynode, local_index);
   return g_tw_lp[index];
}

291
292
293
294
295
296
297
298
299
300
301
/* This function loads the configuration file and sets up the number of LPs on each PE */
void codes_mapping_setup()
{
  int grp, lpt, message_size;
  int pes = tw_nnodes();

  configuration_get_lpgroups(&config, "LPGROUPS", &lpconf);

  for (grp = 0; grp < lpconf.lpgroups_count; grp++)
   {
    for (lpt = 0; lpt < lpconf.lpgroups[grp].lptypes_count; lpt++)
302
	lps_per_pe_floor += (lpconf.lpgroups[grp].lptypes[lpt].count * lpconf.lpgroups[grp].repetitions);
303
   }
304
305
306
  lps_leftover = lps_per_pe_floor % pes;
  lps_per_pe_floor /= pes;
 //printf("\n LPs for this PE are %d reps %d ", lps_per_pe_floor,  lpconf.lpgroups[grp].repetitions);
307
308
309
310
  g_tw_mapping=CUSTOM;
  g_tw_custom_initial_mapping=&codes_mapping_init;
  g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp;
  g_tw_events_per_pe = mem_factor * codes_mapping_get_lps_for_pe();
311
  configuration_get_value_int(&config, "PARAMS", "message_size", NULL, &message_size);
312
313
314
315
316
317
318
319
  if(!message_size)
  {
      message_size = 256;
      printf("\n Warning: ross message size not defined, resetting it to %d", message_size);
  }
  tw_define_lps(codes_mapping_get_lps_for_pe(), message_size, 0);
}

320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
/* given the group and LP type name, return the annotation (or NULL) */
const char* codes_mapping_get_annotation_by_name(char *grp_name, char *lp_type_name){
    int grp, lpt, lp_types_count;
    short found = 0;

    // find the group
    for(grp = 0; grp < lpconf.lpgroups_count; grp++) {
        if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0) {
            found = 1;
            break;
        }
    }

    assert(found);
    found = 0;

    lp_types_count = lpconf.lpgroups[grp].lptypes_count;

    // find the lp type
    for(lpt = 0; lpt < lp_types_count; lpt++) {
        if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0) {
            found = 1;
            break;
        }
    }
    assert(found);

    const char * anno = lpconf.lpgroups[grp].lptypes[lpt].anno;
    if (anno[0]=='\0'){
        return NULL;
    }
    else{
        return anno;
    }
}

const char* codes_mapping_get_annotation_by_lpid(tw_lpid gid){
    int grp, lpt, rep, grp_offset, lp_offset, rep_offset;
    int lp_tmp_id, lp_types_count, lp_count;
    unsigned long grp_lp_count=0;
    short found = 0;

    /* Find the group id first */ 
    for(grp = 0; grp < lpconf.lpgroups_count; grp++)
    {
        grp_offset = 0;
        rep_offset = 0;
        rep = lpconf.lpgroups[grp].repetitions;
        lp_types_count = lpconf.lpgroups[grp].lptypes_count;

        for(lpt = 0; lpt < lp_types_count; lpt++)
        {
            lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
            grp_offset += (rep * lp_count);
            rep_offset += lp_count;
        }
        /* Each gid is assigned an increasing number starting from 0th group and 0th lp type
         * so we check here if the gid lies within the numeric range of a group */ 
        if(gid >= grp_lp_count && gid < grp_lp_count + grp_offset)
        {
            strcpy(local_grp_name, lpconf.lpgroups[grp].name);
            lp_offset = gid - grp_lp_count; /* gets the lp offset starting from the point where the group begins */
            int grp_rep_id = lp_offset / rep_offset;
            lp_tmp_id = lp_offset - (grp_rep_id * rep_offset);
            found = 1;
            break;
        }
        grp_lp_count += grp_offset; /* keep on increasing the group lp count to next group range*/
    }
    assert(found);

    lp_offset = 0;
    found = 0; /* reset found for finding LP type */
    /* Now we compute the LP type ID here based on the lp offset that we just got */ 
    for(lpt = 0; lpt < lp_types_count; lpt++)
    {
        lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
        if(lp_tmp_id >= lp_offset && lp_tmp_id < lp_offset + lp_count)
        {
            found = 1;
            if (lpconf.lpgroups[grp].lptypes[lpt].anno[0] == '\0'){
                return NULL;
            }
            else{
                return lpconf.lpgroups[grp].lptypes[lpt].anno;
            }
        }
        lp_offset += lp_count;
    }
    assert(found);
    return NULL;
}
Philip Carns's avatar
Philip Carns committed
412
413
414
415
416
417
418
419
420

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */