codes_mapping.c 7.78 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5 6 7 8 9 10 11
 */

/* SUMMARY:
 * CODES custom mapping file for ROSS
 */
#include "codes/codes_mapping.h"

12 13
/* number of LPs assigned to the current PE (abstraction of MPI rank) */
static int lps_for_this_pe = 0;
14
static int mem_factor = 1024;
15 16 17 18 19
/* char arrays for holding lp type name and group name*/
char local_grp_name[MAX_NAME_LENGTH], local_lp_name[MAX_NAME_LENGTH];

config_lpgroups_t lpconf;

20 21 22 23 24 25 26 27 28 29 30
int codes_mapping_get_lps_for_pe()
{
  return lps_for_this_pe;
}

/* Takes the global LP ID and returns the rank (PE id) on which the LP is mapped */
tw_peid codes_mapping( tw_lpid gid)
{
  return gid / lps_for_this_pe;
}

31
int codes_mapping_get_group_reps(char* grp_name)
32
{
33 34 35 36 37 38 39 40 41 42
  int grp;
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
  {
     if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
	     return lpconf.lpgroups[grp].repetitions;
  }
  return -1;
}
int codes_mapping_get_lp_count(char* grp_name, char* lp_type_name)
{
43
   int grp, lpt, lp_types_count;
44

45 46
// Account for all lps in the previous groups 
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
47
   {
48 49 50 51 52 53 54 55 56 57
       lp_types_count = lpconf.lpgroups[grp].lptypes_count;
        
       if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
	{
	 for(lpt = 0; lpt < lp_types_count; lpt++)
 	 {
	    if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0)
		return lpconf.lpgroups[grp].lptypes[lpt].count;
	 }		 
	}
58
   }
59
  return -1;
60 61 62 63 64 65 66 67 68
}
/* This function takes the group ID , type ID and rep ID then returns the global LP ID */
/* TODO: Add string based search for LP group and type names */
void codes_mapping_get_lp_id(char* grp_name, char* lp_type_name, int rep_id, int offset, tw_lpid* gid)
{
 int grp, lpt, lpcount = 0, lp_types_count, rep, count_for_this_lpt;
 short found = 0;

 // Account for all lps in the previous groups 
69
 for(grp = 0; grp < lpconf.lpgroups_count; grp++)
70
  {
71 72
    lp_types_count = lpconf.lpgroups[grp].lptypes_count;
    rep = lpconf.lpgroups[grp].repetitions;
73

74
    if(strcmp(lpconf.lpgroups[grp].name, grp_name) == 0)
75 76 77 78 79 80
    {
	    found = 1;
	    break;
    }
    
    for(lpt = 0; lpt < lp_types_count; lpt++)
81
      lpcount += (rep * lpconf.lpgroups[grp].lptypes[lpt].count);
82 83 84 85 86
  }

 assert(found);
 found = 0;

87
 lp_types_count = lpconf.lpgroups[grp].lptypes_count;
88 89 90 91
 
 // Account for the previous lp types in the current repetition
 for(lpt = 0; lpt < lp_types_count; lpt++)
 {
92
   count_for_this_lpt = lpconf.lpgroups[grp].lptypes[lpt].count;
93

94
   if(strcmp(lpconf.lpgroups[grp].lptypes[lpt].name, lp_type_name) == 0)
95 96 97 98 99 100 101 102 103 104 105 106 107
   {
     found = 1;
     break;
   }
   
   lpcount += count_for_this_lpt;
 }

 assert(found);
 // Account for all previous repetitions
 for(rep = 0; rep < rep_id; rep++)
 {
    for(lpt = 0; lpt < lp_types_count; lpt++)
108
      lpcount += lpconf.lpgroups[grp].lptypes[lpt].count;
109 110 111 112 113 114 115 116 117 118 119 120 121
 }
   *gid = lpcount + offset;
}

/* This function takes the LP ID and returns its grp index, lp type ID and repetition ID */
void codes_mapping_get_lp_info(tw_lpid gid, char* grp_name, int* grp_id, int* lp_type_id, char* lp_type_name, int* grp_rep_id, int* offset)
{
  int grp, lpt, rep, grp_offset, lp_offset, rep_offset;
  int lp_tmp_id, lp_types_count, lp_count;
  unsigned long grp_lp_count=0;
  short found = 0;
 
  /* Find the group id first */ 
122
  for(grp = 0; grp < lpconf.lpgroups_count; grp++)
123 124 125
  {
    grp_offset = 0;
    rep_offset = 0;
126 127
    rep = lpconf.lpgroups[grp].repetitions;
    lp_types_count = lpconf.lpgroups[grp].lptypes_count;
128 129 130

    for(lpt = 0; lpt < lp_types_count; lpt++)
    {
131
	lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
132 133 134 135 136 137 138 139
	grp_offset += (rep * lp_count);
	rep_offset += lp_count;
    }
    /* Each gid is assigned an increasing number starting from 0th group and 0th lp type
     * so we check here if the gid lies within the numeric range of a group */ 
    if(gid >= grp_lp_count && gid < grp_lp_count + grp_offset)
    {
	*grp_id = grp;
140
	 strcpy(local_grp_name, lpconf.lpgroups[grp].name);
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
	 lp_offset = gid - grp_lp_count; /* gets the lp offset starting from the point where the group begins */
         *grp_rep_id = lp_offset / rep_offset;
          lp_tmp_id = lp_offset - (*grp_rep_id * rep_offset);
	  found = 1;
	 break;
    }
    grp_lp_count += grp_offset; /* keep on increasing the group lp count to next group range*/
  }
  assert(found);

  lp_offset = 0;
  found = 0; /* reset found for finding LP type */
 /* Now we compute the LP type ID here based on the lp offset that we just got */ 
  for(lpt = 0; lpt < lp_types_count; lpt++)
  {
156
     lp_count = lpconf.lpgroups[grp].lptypes[lpt].count;
157 158 159
     if(lp_tmp_id >= lp_offset && lp_tmp_id < lp_offset + lp_count)
     {
	     *lp_type_id = lpt;
160
	      strcpy(local_lp_name, lpconf.lpgroups[grp].lptypes[lpt].name);
161 162 163 164 165 166 167 168 169 170 171 172 173
	      *offset = lp_tmp_id - lp_offset;
	      found = 1;
     	      break;
     }
     lp_offset += lp_count;
  }
  assert(found);
  strncpy(grp_name, local_grp_name, MAX_NAME_LENGTH);
  strncpy(lp_type_name, local_lp_name, MAX_NAME_LENGTH);
  //printf("\n gid %d lp type name %s rep_id %d ", gid, lp_type_name, *grp_rep_id);
}

/* This function assigns local and global LP Ids to LPs */
174
static void codes_mapping_init(void)
175 176 177 178 179 180
{
     int grp_id, lpt_id, rep_id, offset;
     tw_lpid ross_gid, ross_lid; /* ross global and local IDs */
     tw_pe * pe;
     char lp_type_name[MAX_NAME_LENGTH];
     char grp_name[MAX_NAME_LENGTH];
181
     int nkp_per_pe = g_tw_nkp;
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
     tw_lpid         lpid, kpid;

     /* have 16 kps per pe, this is the optimized configuration for ROSS custom mapping */
     for(kpid = 0; kpid < nkp_per_pe; kpid++)
	tw_kp_onpe(kpid, g_tw_pe[0]);

     int lp_init_range = g_tw_mynode * lps_for_this_pe;
     codes_mapping_get_lp_info(lp_init_range, grp_name, &grp_id, &lpt_id, lp_type_name, &rep_id, &offset);

     for (lpid = lp_init_range; lpid < lp_init_range + lps_for_this_pe; lpid++)
      {
	 ross_gid = lpid;
	 ross_lid = lpid - lp_init_range;
	 kpid = ross_lid % g_tw_nkp;
	 pe = tw_getpe(kpid % g_tw_npe);
	 codes_mapping_get_lp_info(ross_gid, grp_name, &grp_id, &lpt_id, lp_type_name, &rep_id, &offset);
	 tw_lp_onpe(ross_lid , pe, ross_gid);
	 tw_lp_onkp(g_tw_lp[ross_lid], g_tw_kp[kpid]);
	 tw_lp_settype(ross_lid, lp_type_lookup(lp_type_name));
     }
     return;
}

/* This function takes the global LP ID, maps it to the local LP ID and returns the LP 
 * lps have global and local LP IDs
 * global LP IDs are unique across all PEs, local LP IDs are unique within a PE */
208
static tw_lp * codes_mapping_to_lp( tw_lpid lpid)
209 210 211 212 213 214
{
   int index = lpid - (g_tw_mynode * lps_for_this_pe);
//   printf("\n global id %d index %d lps_before %d lps_offset %d local index %d ", lpid, index, lps_before, g_tw_mynode, local_index);
   return g_tw_lp[index];
}

215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
/* This function loads the configuration file and sets up the number of LPs on each PE */
void codes_mapping_setup()
{
  int grp, lpt, message_size;
  int pes = tw_nnodes();

  configuration_get_lpgroups(&config, "LPGROUPS", &lpconf);

  for (grp = 0; grp < lpconf.lpgroups_count; grp++)
   {
    for (lpt = 0; lpt < lpconf.lpgroups[grp].lptypes_count; lpt++)
	lps_for_this_pe += (lpconf.lpgroups[grp].lptypes[lpt].count * lpconf.lpgroups[grp].repetitions);
   }
  lps_for_this_pe /= pes;
 //printf("\n LPs for this PE are %d reps %d ", lps_for_this_pe,  lpconf.lpgroups[grp].repetitions);
  g_tw_mapping=CUSTOM;
  g_tw_custom_initial_mapping=&codes_mapping_init;
  g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp;
  g_tw_events_per_pe = mem_factor * codes_mapping_get_lps_for_pe();
  configuration_get_value_int(&config, "PARAMS", "message_size", &message_size);
  if(!message_size)
  {
      message_size = 256;
      printf("\n Warning: ross message size not defined, resetting it to %d", message_size);
  }
  tw_define_lps(codes_mapping_get_lps_for_pe(), message_size, 0);
}

Philip Carns's avatar
Philip Carns committed
243 244 245 246 247 248 249 250 251

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */