codes_mapping.c 13.7 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5 6 7 8 9 10 11
 */

/* SUMMARY:
 * CODES custom mapping file for ROSS
 */
#include "codes/codes_mapping.h"

12 13
#define CODES_MAPPING_DEBUG 0

14 15 16 17 18 19
/* number of LPs assigned to the current PE (abstraction of MPI rank).
 * for lp counts which are not divisible by the number of ranks, keep 
 * modulus around */
static int lps_per_pe_floor = 0;
static int lps_leftover = 0;

Jonathan Jenkins's avatar
Jonathan Jenkins committed
20
static int mem_factor = 256;
21 22 23

static inline int mini(int a, int b){ return a < b ? a : b; }

24 25 26 27 28
/* char arrays for holding lp type name and group name*/
char local_grp_name[MAX_NAME_LENGTH], local_lp_name[MAX_NAME_LENGTH];

config_lpgroups_t lpconf;

29 30
int codes_mapping_get_lps_for_pe()
{
31 32 33 34 35 36
    int rank;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#if CODES_MAPPING_DEBUG
    printf("%d lps for rank %d\n", lps_per_pe_floor+(g_tw_mynode < lps_leftover), rank);
#endif
  return lps_per_pe_floor + (g_tw_mynode < lps_leftover);
37 38 39 40 41
}

/* Takes the global LP ID and returns the rank (PE id) on which the LP is mapped */
tw_peid codes_mapping( tw_lpid gid)
{
42 43 44 45 46 47 48 49
    int lps_on_pes_with_leftover = lps_leftover * (lps_per_pe_floor+1);
    if (gid < lps_on_pes_with_leftover){
        return gid / (lps_per_pe_floor+1);
    }
    else{
        return (gid-lps_on_pes_with_leftover)/lps_per_pe_floor + lps_leftover;
    }
  /*return gid / lps_per_pe_floor;*/
50 51
}

52
int codes_mapping_get_group_reps(char* grp_name)
53
{
54 55 56 57 58 59 60 61 62 63
  int grp;
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
  {
     if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
	     return lpconf.lpgroups[grp].repetitions;
  }
  return -1;
}
int codes_mapping_get_lp_count(char* grp_name, char* lp_type_name)
{
64
   int grp, lpt, lp_types_count;
65

66 67
// Account for all lps in the previous groups 
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
68
   {
69 70 71 72 73 74 75 76 77 78
       lp_types_count = lpconf.lpgroups[grp].lptypes_count;
        
       if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
	{
	 for(lpt = 0; lpt < lp_types_count; lpt++)
 	 {
	    if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0)
		return lpconf.lpgroups[grp].lptypes[lpt].count;
	 }		 
	}
79
   }
80
  return -1;
81
}
82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99

int codes_mapping_get_global_lp_count(char* lp_type_name){
    int total_ct = 0;
    int grp;
    for (grp = 0; grp < lpconf.lpgroups_count; grp++){
        int lpt;
        int lp_types_count = lpconf.lpgroups[grp].lptypes_count;
        int reps = lpconf.lpgroups[grp].repetitions;
        config_lptype_t *c = lpconf.lpgroups[grp].lptypes;
        for (lpt = 0; lpt < lp_types_count; lpt++){
            if(strcmp(c[lpt].name, lp_type_name) == 0){
                total_ct += c[lpt].count * reps;
            }
        }
    }
    return total_ct;
}

100 101 102 103 104 105 106 107
/* This function takes the group ID , type ID and rep ID then returns the global LP ID */
/* TODO: Add string based search for LP group and type names */
void codes_mapping_get_lp_id(char* grp_name, char* lp_type_name, int rep_id, int offset, tw_lpid* gid)
{
 int grp, lpt, lpcount = 0, lp_types_count, rep, count_for_this_lpt;
 short found = 0;

 // Account for all lps in the previous groups 
108
 for(grp = 0; grp < lpconf.lpgroups_count; grp++)
109
  {
110 111
    lp_types_count = lpconf.lpgroups[grp].lptypes_count;
    rep = lpconf.lpgroups[grp].repetitions;
112

113
    if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
114 115 116 117 118 119
    {
	    found = 1;
	    break;
    }
    
    for(lpt = 0; lpt < lp_types_count; lpt++)
120
      lpcount += (rep * lpconf.lpgroups[grp].lptypes[lpt].count);
121 122 123 124 125
  }

 assert(found);
 found = 0;

126
 lp_types_count = lpconf.lpgroups[grp].lptypes_count;
127 128 129 130
 
 // Account for the previous lp types in the current repetition
 for(lpt = 0; lpt < lp_types_count; lpt++)
 {
131
   count_for_this_lpt = lpconf.lpgroups[grp].lptypes[lpt].count;
132

133
   if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0)
134 135 136 137 138 139 140 141 142 143 144 145 146
   {
     found = 1;
     break;
   }
   
   lpcount += count_for_this_lpt;
 }

 assert(found);
 // Account for all previous repetitions
 for(rep = 0; rep < rep_id; rep++)
 {
    for(lpt = 0; lpt < lp_types_count; lpt++)
147
      lpcount += lpconf.lpgroups[grp].lptypes[lpt].count;
148
 }
149
   *gid = lpcount + offset % count_for_this_lpt;
150 151
}

152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
int codes_mapping_get_lp_global_rel_id(tw_lpid gid){
    /* use mapping function to get our starting point */
    char grp_name[MAX_NAME_LENGTH];
    char lp_type_name[MAX_NAME_LENGTH];
    int grp_id, lp_type_id, grp_rep_id, offset;
    codes_mapping_get_lp_info(gid, grp_name, &grp_id, &lp_type_id, lp_type_name,
            &grp_rep_id, &offset);

    /* now, go through the groups that occurred before and count up the lps of
     * the same name */
    int grp;
    int total_lp_count = 0;
    for (grp = 0; grp < grp_id; grp++){
        int rep = lpconf.lpgroups[grp].repetitions;
        int l;
        /* get the total number of LPs in this group */
        for (l = 0; l < lpconf.lpgroups[grp].lptypes_count; l++){
            if (strcmp(lp_type_name, lpconf.lpgroups[grp].lptypes[l].name) == 0){
                total_lp_count += rep * lpconf.lpgroups[grp].lptypes[l].count;
            }
        }
    }

    /* the global relative lp id is my relative position within the group +
     * the number of lps that came before */
    int nlp_per_rep = lpconf.lpgroups[grp_id].lptypes[lp_type_id].count;
    return total_lp_count + (grp_rep_id * nlp_per_rep) + offset;
}


182 183 184 185 186 187 188 189 190
/* This function takes the LP ID and returns its grp index, lp type ID and repetition ID */
void codes_mapping_get_lp_info(tw_lpid gid, char* grp_name, int* grp_id, int* lp_type_id, char* lp_type_name, int* grp_rep_id, int* offset)
{
  int grp, lpt, rep, grp_offset, lp_offset, rep_offset;
  int lp_tmp_id, lp_types_count, lp_count;
  unsigned long grp_lp_count=0;
  short found = 0;
 
  /* Find the group id first */ 
191
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
192 193 194
  {
    grp_offset = 0;
    rep_offset = 0;
195 196
    rep = lpconf.lpgroups[grp].repetitions;
    lp_types_count = lpconf.lpgroups[grp].lptypes_count;
197 198 199

    for(lpt = 0; lpt < lp_types_count; lpt++)
    {
200
	lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
201 202 203 204 205 206 207 208
	grp_offset += (rep * lp_count);
	rep_offset += lp_count;
    }
    /* Each gid is assigned an increasing number starting from 0th group and 0th lp type
     * so we check here if the gid lies within the numeric range of a group */ 
    if(gid >= grp_lp_count && gid < grp_lp_count + grp_offset)
    {
	*grp_id = grp;
209
	 strcpy(local_grp_name, lpconf.lpgroups[grp].name);
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
	 lp_offset = gid - grp_lp_count; /* gets the lp offset starting from the point where the group begins */
         *grp_rep_id = lp_offset / rep_offset;
          lp_tmp_id = lp_offset - (*grp_rep_id * rep_offset);
	  found = 1;
	 break;
    }
    grp_lp_count += grp_offset; /* keep on increasing the group lp count to next group range*/
  }
  assert(found);

  lp_offset = 0;
  found = 0; /* reset found for finding LP type */
 /* Now we compute the LP type ID here based on the lp offset that we just got */ 
  for(lpt = 0; lpt < lp_types_count; lpt++)
  {
225
     lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
226 227 228
     if(lp_tmp_id >= lp_offset && lp_tmp_id < lp_offset + lp_count)
     {
	     *lp_type_id = lpt;
229
	      strcpy(local_lp_name, lpconf.lpgroups[grp].lptypes[lpt].name);
230 231 232 233 234 235 236 237 238 239 240 241 242
	      *offset = lp_tmp_id - lp_offset;
	      found = 1;
     	      break;
     }
     lp_offset += lp_count;
  }
  assert(found);
  strncpy(grp_name, local_grp_name, MAX_NAME_LENGTH);
  strncpy(lp_type_name, local_lp_name, MAX_NAME_LENGTH);
  //printf("\n gid %d lp type name %s rep_id %d ", gid, lp_type_name, *grp_rep_id);
}

/* This function assigns local and global LP Ids to LPs */
243
static void codes_mapping_init(void)
244 245 246 247 248 249
{
     int grp_id, lpt_id, rep_id, offset;
     tw_lpid ross_gid, ross_lid; /* ross global and local IDs */
     tw_pe * pe;
     char lp_type_name[MAX_NAME_LENGTH];
     char grp_name[MAX_NAME_LENGTH];
250
     int nkp_per_pe = g_tw_nkp;
251 252 253 254 255 256
     tw_lpid         lpid, kpid;

     /* have 16 kps per pe, this is the optimized configuration for ROSS custom mapping */
     for(kpid = 0; kpid < nkp_per_pe; kpid++)
	tw_kp_onpe(kpid, g_tw_pe[0]);

257 258 259 260 261
     int lp_start =
         g_tw_mynode * lps_per_pe_floor + mini(g_tw_mynode,lps_leftover);
     int lp_end =
         (g_tw_mynode+1) * lps_per_pe_floor + mini(g_tw_mynode+1,lps_leftover);
     codes_mapping_get_lp_info(lp_start, grp_name, &grp_id, &lpt_id, lp_type_name, &rep_id, &offset);
262

263
     for (lpid = lp_start; lpid < lp_end; lpid++)
264 265
      {
	 ross_gid = lpid;
266
	 ross_lid = lpid - lp_start;
267 268 269
	 kpid = ross_lid % g_tw_nkp;
	 pe = tw_getpe(kpid % g_tw_npe);
	 codes_mapping_get_lp_info(ross_gid, grp_name, &grp_id, &lpt_id, lp_type_name, &rep_id, &offset);
270 271 272
#if CODES_MAPPING_DEBUG
         printf("lp:%lu --> kp:%lu, pe:%llu\n", ross_gid, kpid, pe->id);
#endif
273
	 tw_lp_onpe(ross_lid, pe, ross_gid);
274 275 276 277 278 279 280 281 282
	 tw_lp_onkp(g_tw_lp[ross_lid], g_tw_kp[kpid]);
	 tw_lp_settype(ross_lid, lp_type_lookup(lp_type_name));
     }
     return;
}

/* This function takes the global LP ID, maps it to the local LP ID and returns the LP 
 * lps have global and local LP IDs
 * global LP IDs are unique across all PEs, local LP IDs are unique within a PE */
283
static tw_lp * codes_mapping_to_lp( tw_lpid lpid)
284
{
285 286
   int index = lpid - (g_tw_mynode * lps_per_pe_floor) -
       mini(g_tw_mynode, lps_leftover);
287 288 289 290
//   printf("\n global id %d index %d lps_before %d lps_offset %d local index %d ", lpid, index, lps_before, g_tw_mynode, local_index);
   return g_tw_lp[index];
}

291 292 293 294 295 296 297 298 299 300 301
/* This function loads the configuration file and sets up the number of LPs on each PE */
void codes_mapping_setup()
{
  int grp, lpt, message_size;
  int pes = tw_nnodes();

  configuration_get_lpgroups(&config, "LPGROUPS", &lpconf);

  for (grp = 0; grp < lpconf.lpgroups_count; grp++)
   {
    for (lpt = 0; lpt < lpconf.lpgroups[grp].lptypes_count; lpt++)
302
	lps_per_pe_floor += (lpconf.lpgroups[grp].lptypes[lpt].count * lpconf.lpgroups[grp].repetitions);
303
   }
304 305 306
  lps_leftover = lps_per_pe_floor % pes;
  lps_per_pe_floor /= pes;
 //printf("\n LPs for this PE are %d reps %d ", lps_per_pe_floor,  lpconf.lpgroups[grp].repetitions);
307 308 309 310
  g_tw_mapping=CUSTOM;
  g_tw_custom_initial_mapping=&codes_mapping_init;
  g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp;
  g_tw_events_per_pe = mem_factor * codes_mapping_get_lps_for_pe();
311
  configuration_get_value_int(&config, "PARAMS", "message_size", NULL, &message_size);
312 313 314 315 316 317 318 319
  if(!message_size)
  {
      message_size = 256;
      printf("\n Warning: ross message size not defined, resetting it to %d", message_size);
  }
  tw_define_lps(codes_mapping_get_lps_for_pe(), message_size, 0);
}

320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411
/* given the group and LP type name, return the annotation (or NULL) */
const char* codes_mapping_get_annotation_by_name(char *grp_name, char *lp_type_name){
    int grp, lpt, lp_types_count;
    short found = 0;

    // find the group
    for(grp = 0; grp < lpconf.lpgroups_count; grp++) {
        if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0) {
            found = 1;
            break;
        }
    }

    assert(found);
    found = 0;

    lp_types_count = lpconf.lpgroups[grp].lptypes_count;

    // find the lp type
    for(lpt = 0; lpt < lp_types_count; lpt++) {
        if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0) {
            found = 1;
            break;
        }
    }
    assert(found);

    const char * anno = lpconf.lpgroups[grp].lptypes[lpt].anno;
    if (anno[0]=='\0'){
        return NULL;
    }
    else{
        return anno;
    }
}

const char* codes_mapping_get_annotation_by_lpid(tw_lpid gid){
    int grp, lpt, rep, grp_offset, lp_offset, rep_offset;
    int lp_tmp_id, lp_types_count, lp_count;
    unsigned long grp_lp_count=0;
    short found = 0;

    /* Find the group id first */ 
    for(grp = 0; grp < lpconf.lpgroups_count; grp++)
    {
        grp_offset = 0;
        rep_offset = 0;
        rep = lpconf.lpgroups[grp].repetitions;
        lp_types_count = lpconf.lpgroups[grp].lptypes_count;

        for(lpt = 0; lpt < lp_types_count; lpt++)
        {
            lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
            grp_offset += (rep * lp_count);
            rep_offset += lp_count;
        }
        /* Each gid is assigned an increasing number starting from 0th group and 0th lp type
         * so we check here if the gid lies within the numeric range of a group */ 
        if(gid >= grp_lp_count && gid < grp_lp_count + grp_offset)
        {
            strcpy(local_grp_name, lpconf.lpgroups[grp].name);
            lp_offset = gid - grp_lp_count; /* gets the lp offset starting from the point where the group begins */
            int grp_rep_id = lp_offset / rep_offset;
            lp_tmp_id = lp_offset - (grp_rep_id * rep_offset);
            found = 1;
            break;
        }
        grp_lp_count += grp_offset; /* keep on increasing the group lp count to next group range*/
    }
    assert(found);

    lp_offset = 0;
    found = 0; /* reset found for finding LP type */
    /* Now we compute the LP type ID here based on the lp offset that we just got */ 
    for(lpt = 0; lpt < lp_types_count; lpt++)
    {
        lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
        if(lp_tmp_id >= lp_offset && lp_tmp_id < lp_offset + lp_count)
        {
            found = 1;
            if (lpconf.lpgroups[grp].lptypes[lpt].anno[0] == '\0'){
                return NULL;
            }
            else{
                return lpconf.lpgroups[grp].lptypes[lpt].anno;
            }
        }
        lp_offset += lp_count;
    }
    assert(found);
    return NULL;
}
Philip Carns's avatar
Philip Carns committed
412

413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
/*
 * Returns a mapping of LP name to all annotations used with the type
 *
 * lp_name     - lp name as used in the configuration
 */
const config_anno_map_t * 
codes_mapping_get_lp_anno_map(const char *lp_name){
    for (uint64_t i = 0; i < lpconf.lpannos_count; i++){
        if (strcmp(lp_name, lpconf.lpannos[i].lp_name) == 0){
            return &lpconf.lpannos[i];
        }
    }
    return NULL;
}

Philip Carns's avatar
Philip Carns committed
428 429 430 431 432 433 434 435
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */