/* * Copyright (C) 2013 University of Chicago. * See COPYRIGHT notice in top-level directory. * */ /* SUMMARY: * CODES custom mapping file for ROSS */ #include "codes/codes_mapping.h" #define CODES_MAPPING_DEBUG 0 /* number of LPs assigned to the current PE (abstraction of MPI rank). * for lp counts which are not divisible by the number of ranks, keep * modulus around */ static tw_lpid lps_per_pe_floor = 0; static tw_lpid lps_leftover = 0; static int mem_factor = 256; static int mini(int a, int b){ return a < b ? a : b; } // compare passed in annotation strings (NULL or nonempty) against annotation // strings in the config (empty or nonempty) static int cmp_anno(const char * anno_user, const char * anno_config){ return anno_user == NULL ? anno_config == NULL : (anno_config != NULL && !strcmp(anno_user, anno_config)); } #if 0 // TODO: this code seems useful, but I'm not sure where to put it for the time // being. It should be useful when we are directly reading from the // configuration file, which has the unprocessed annotated strings // return code matches that of strcmp and friends, checks // lp_type_name@annotation against full name // NOTE: no argument should be NULL or invalid strings static int strcmp_anno( const char * full_name, const char * prefix, const char * annotation){ int i; // first phase - check full_name against prefix for (i = 0; i < MAX_NAME_LENGTH; i++){ char cf = full_name[i], cl = prefix[i]; if (cl == '\0') break; // match successful else if (cf != cl) // captures case where cf is null char or @ return (int)(cf-cl); // lp name on full doesn't match } if (i==MAX_NAME_LENGTH) return 1; // second phase - ensure the next character after the match is an @ if (full_name[i] != '@') return -1; // third phase, compare the remaining full_name against annotation return strcmp(full_name+i+1, annotation); } #endif /* char arrays for holding lp type name and group name*/ // (unused var atm) static char local_group_name[MAX_NAME_LENGTH]; static char local_lp_name[MAX_NAME_LENGTH], local_annotation[MAX_NAME_LENGTH]; int codes_mapping_get_lps_for_pe() { int rank; MPI_Comm_rank(MPI_COMM_CODES, &rank); #if CODES_MAPPING_DEBUG printf("%d lps for rank %d\n", lps_per_pe_floor+(g_tw_mynode < lps_leftover), rank); #endif return lps_per_pe_floor + ((tw_lpid)g_tw_mynode < lps_leftover); } /* Takes the global LP ID and returns the rank (PE id) on which the LP is mapped */ tw_peid codes_mapping( tw_lpid gid) { tw_lpid lps_on_pes_with_leftover = lps_leftover * (lps_per_pe_floor+1); if (gid < lps_on_pes_with_leftover){ return gid / (lps_per_pe_floor+1); } else{ return (gid-lps_on_pes_with_leftover)/lps_per_pe_floor + lps_leftover; } /*return gid / lps_per_pe_floor;*/ } int codes_mapping_get_group_reps(const char* group_name) { int grp; for(grp = 0; grp < lpconf.lpgroups_count; grp++) { if(strcmp(lpconf.lpgroups[grp].name.ptr, group_name) == 0) return lpconf.lpgroups[grp].repetitions; } return -1; } int codes_mapping_get_lp_count( const char * group_name, int ignore_repetitions, const char * lp_type_name, const char * annotation, int ignore_annos){ int lp_type_ct_total = 0; // check - if group name is null, then disable ignore_repetitions (the // former takes precedence) if (group_name == NULL) ignore_repetitions = 0; for (int g = 0; g < lpconf.lpgroups_count; g++){ const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; // iterate over the lps if the group is null (count across all groups) // or if the group names match if (group_name == NULL || strcmp(lpg->name.ptr, group_name) == 0){ for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; if (strcmp(lp_type_name, lpt->name.ptr) == 0){ // increment the count if we are ignoring annotations, // query and entry are both unannotated, or if the // annotations match if (ignore_annos || cmp_anno(annotation, lpt->anno.ptr)){ if (ignore_repetitions) lp_type_ct_total += lpt->count; else lp_type_ct_total += lpt->count * lpg->repetitions; if (ignore_annos == 1) break; } } } if (group_name != NULL) break; // we've processed the exact group } } // no error needed here - 0 is a valid value return lp_type_ct_total; } void codes_mapping_get_lp_id( const char * group_name, const char * lp_type_name, const char * annotation, int ignore_anno, int rep_id, int offset, tw_lpid * gid){ // sanity checks if (rep_id < 0 || offset < 0 || group_name == NULL || lp_type_name == NULL) goto ERROR; *gid = 0; // for each group for (int g = 0; g < lpconf.lpgroups_count; g++){ const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; // in any case, count up the lps (need for repetition handling) tw_lpid rep_count = 0; for (int l = 0; l < lpg->lptypes_count; l++){ rep_count += lpg->lptypes[l].count; } // does group name match? if (strcmp(lpg->name.ptr, group_name) == 0){ // sanity check rep_id if (rep_id >= lpg->repetitions) goto ERROR; tw_lpid local_lp_count = 0; // for each lp type for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; // does lp name match? if (strcmp(lpt->name.ptr, lp_type_name) == 0){ // does annotation match (or are we ignoring annotations?) if (ignore_anno || cmp_anno(annotation, lpt->anno.ptr)){ // return if sane offset if (offset >= (int) lpt->count){ goto ERROR; } else{ *gid += (rep_count * (tw_lpid)rep_id) + local_lp_count + (tw_lpid)offset; return; } } } local_lp_count += lpt->count; } // after a group match, short circuit to end if lp not found goto ERROR; } // group name doesn't match, add count to running total and move on else{ *gid += lpg->repetitions * rep_count; } } ERROR: // LP not found tw_error(TW_LOC, "Unable to find LP id given " "group \"%s\", " "typename \"%s\", " "annotation \"%s\", " "repetition %d, and offset %d", group_name==NULL ? "" : group_name, lp_type_name==NULL ? "" : lp_type_name, annotation==NULL ? "" : annotation, rep_id, offset); } int codes_mapping_get_lp_relative_id( tw_lpid gid, int group_wise, int annotation_wise){ int group_index, lp_type_index, rep_id, offset; codes_mapping_get_lp_info(gid, NULL, &group_index, local_lp_name, &lp_type_index, local_annotation, &rep_id, &offset); const char * anno = (local_annotation[0]=='\0') ? NULL : local_annotation; int group_lp_count = 0; // if not group_specific, then count up LPs of all preceding groups if (!group_wise){ for (int g = 0; g < group_index; g++){ int lp_count = 0; const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; if (strcmp(local_lp_name, lpt->name.ptr) == 0){ // increment the count if we are ignoring annotations, // both LPs are unannotated, or if the annotations match if (!annotation_wise || cmp_anno(anno, lpt->anno.ptr)){ lp_count += lpt->count; } } } group_lp_count += lp_count * lpg->repetitions; } } // count up LPs within my group occuring before me // (the loop is necessary because different annotated LPs may exist in the // same group) int lp_count = 0; int lp_pre_count = 0; for (int l = 0; l < lpconf.lpgroups[group_index].lptypes_count; l++){ const config_lptype_t *lpt = &lpconf.lpgroups[group_index].lptypes[l]; if (strcmp(local_lp_name, lpt->name.ptr) == 0){ if (!annotation_wise || cmp_anno(anno, lpt->anno.ptr)){ lp_count += lpt->count; // special case: if we find an LP entry that matches, but is not // the same entry where the input gid comes from, then increment // a separate "pre" counter if (l < lp_type_index){ lp_pre_count += lpt->count; } } } } // now we have the necessary counts // return lps in groups that came before + // lps within the group that came before the target LP return (int) (group_lp_count + (lp_count * rep_id) + lp_pre_count + offset); } tw_lpid codes_mapping_get_lpid_from_relative( int relative_id, const char * group_name, const char * lp_type_name, const char * annotation, int annotation_wise){ // strategy: count up all preceding LPs. When we reach a point where an LP // type matches, count up the relative ID. When the accumulated relative ID // matches or surpasses the input, then we've found our LP int rel_id_count = 0; tw_lpid gid_count = 0; for (int g = 0; g < lpconf.lpgroups_count; g++){ const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; if (group_name == NULL || strcmp(group_name, lpg->name.ptr) == 0){ // consider this group for counting tw_lpid local_gid_count = 0; int local_rel_id_count = 0; for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; local_gid_count += lpt->count; if (strcmp(lp_type_name, lpt->name.ptr) == 0 && (!annotation_wise || cmp_anno(annotation, lpt->anno.ptr))){ local_rel_id_count += lpt->count; } } // is our relative id within this group? if (relative_id < rel_id_count + lpg->repetitions * local_rel_id_count){ tw_lpid gid = gid_count; int rem = relative_id - rel_id_count; int rep = rem / local_rel_id_count; rem -= (rep * local_rel_id_count); gid += local_gid_count * rep; // count up lps listed prior to this entry for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; if ( strcmp(lp_type_name, lpt->name.ptr) == 0 && (!annotation_wise || cmp_anno(annotation, lpt->anno.ptr))){ if (rem < (int) lpt->count){ return gid + (tw_lpid) rem; } else{ rem -= lpt->count; } } gid += lpt->count; } // this shouldn't happen goto NOT_FOUND; } else if (group_name != NULL){ // immediate error - found the group, but not the id goto NOT_FOUND; } else{ // increment and move to the next group rel_id_count += local_rel_id_count * lpg->repetitions; gid_count += local_gid_count * lpg->repetitions; } } else{ // just count up the group LPs tw_lpid local_gid_count = 0; for (int l = 0; l < lpg->lptypes_count; l++){ local_gid_count += lpg->lptypes[l].count; } gid_count += local_gid_count * lpg->repetitions; } } NOT_FOUND: tw_error(TW_LOC, "Unable to find LP-ID for ID %d relative to group %s, " "lp name %s, annotation %s", relative_id, group_name == NULL ? "(all)" : group_name, lp_type_name, annotation_wise ? annotation : "(all)"); return 0; // dummy return } void codes_mapping_get_lp_info( tw_lpid gid, char * group_name, int * group_index, char * lp_type_name, int * lp_type_index, char * annotation, int * rep_id, int * offset){ // running total of lp's we've seen so far tw_lpid id_total = 0; // for each group for (int g = 0; g < lpconf.lpgroups_count; g++){ const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; tw_lpid num_id_group, num_id_per_rep = 0; // count up the number of ids in this group for (int l = 0; l < lpg->lptypes_count; l++){ num_id_per_rep += lpg->lptypes[l].count; } num_id_group = num_id_per_rep * lpg->repetitions; if (num_id_group+id_total > gid){ // we've found the group tw_lpid rem = gid - id_total; if (group_name != NULL) strcpy(group_name, lpg->name.ptr); *group_index = g; // find repetition within group *rep_id = (int) (rem / num_id_per_rep); rem -= num_id_per_rep * (tw_lpid)*rep_id; num_id_per_rep = 0; for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; if (rem < num_id_per_rep + lpt->count){ // found the specific LP if (lp_type_name != NULL) strcpy(lp_type_name, lpt->name.ptr); if (annotation != NULL) { if (lpt->anno.ptr == NULL) annotation[0] = '\0'; else strcpy(annotation, lpt->anno.ptr); } *offset = (int) (rem - num_id_per_rep); *lp_type_index = l; return; // done } else{ num_id_per_rep += lpg->lptypes[l].count; } } } else{ id_total += num_id_group; } } // LP not found tw_error(TW_LOC, "Unable to find LP info given gid %lu", gid); } void codes_mapping_get_lp_info2( tw_lpid gid, char const * * group_name, char const * * lp_type_name, char const * * annotation, int * rep_id, int * offset) { // running total of lp's we've seen so far tw_lpid id_total = 0; // for each group for (int g = 0; g < lpconf.lpgroups_count; g++){ const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; tw_lpid num_id_group, num_id_per_rep = 0; // count up the number of ids in this group for (int l = 0; l < lpg->lptypes_count; l++){ num_id_per_rep += lpg->lptypes[l].count; } num_id_group = num_id_per_rep * lpg->repetitions; if (num_id_group+id_total > gid){ // we've found the group tw_lpid rem = gid - id_total; if (group_name != NULL) *group_name = lpg->name.ptr; // find repetition within group *rep_id = (int) (rem / num_id_per_rep); rem -= num_id_per_rep * (tw_lpid)*rep_id; num_id_per_rep = 0; for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; if (rem < num_id_per_rep + lpt->count){ // found the specific LP if (lp_type_name != NULL) *lp_type_name = lpt->name.ptr; if (annotation != NULL) *annotation = lpt->anno.ptr; *offset = (int) (rem - num_id_per_rep); return; // done } else{ num_id_per_rep += lpg->lptypes[l].count; } } } else{ id_total += num_id_group; } } // LP not found tw_error(TW_LOC, "Unable to find LP info given gid %lu", gid); } /* This function assigns local and global LP Ids to LPs */ static void codes_mapping_init(void) { int grp_id, lpt_id, rep_id, offset; tw_lpid ross_gid, ross_lid; /* ross global and local IDs */ tw_pe * pe; char lp_type_name[MAX_NAME_LENGTH]; tw_lpid nkp_per_pe = g_tw_nkp; tw_lpid lpid, kpid; const tw_lptype *lptype; const st_trace_type *trace_type; /* have 16 kps per pe, this is the optimized configuration for ROSS custom mapping */ for(kpid = 0; kpid < nkp_per_pe; kpid++) tw_kp_onpe(kpid, g_tw_pe[0]); tw_lpid lp_start = g_tw_mynode * lps_per_pe_floor + mini(g_tw_mynode,lps_leftover); tw_lpid lp_end = (g_tw_mynode+1) * lps_per_pe_floor + mini(g_tw_mynode+1,lps_leftover); for (lpid = lp_start; lpid < lp_end; lpid++) { ross_gid = lpid; ross_lid = lpid - lp_start; kpid = ross_lid % g_tw_nkp; pe = tw_getpe(kpid % g_tw_npe); codes_mapping_get_lp_info(ross_gid, NULL, &grp_id, lp_type_name, &lpt_id, NULL, &rep_id, &offset); #if CODES_MAPPING_DEBUG printf("lp:%lu --> kp:%lu, pe:%llu\n", ross_gid, kpid, pe->id); #endif tw_lp_onpe(ross_lid, pe, ross_gid); tw_lp_onkp(g_tw_lp[ross_lid], g_tw_kp[kpid]); lptype = lp_type_lookup(lp_type_name); if (lptype == NULL) tw_error(TW_LOC, "could not find LP with type name \"%s\", " "did you forget to register the LP?\n", lp_type_name); else /* sorry, const... */ tw_lp_settype(ross_lid, (tw_lptype*) lptype); if (g_st_ev_trace) { trace_type = trace_type_lookup(lp_type_name); st_evtrace_settype(ross_lid, (st_trace_type*) trace_type); } } return; } /* This function takes the global LP ID, maps it to the local LP ID and returns the LP * lps have global and local LP IDs * global LP IDs are unique across all PEs, local LP IDs are unique within a PE */ static tw_lp * codes_mapping_to_lp( tw_lpid lpid) { int index = lpid - (g_tw_mynode * lps_per_pe_floor) - mini(g_tw_mynode, lps_leftover); // printf("\n global id %d index %d lps_before %d lps_offset %d local index %d ", lpid, index, lps_before, g_tw_mynode, local_index); return g_tw_lp[index]; } /* This function loads the configuration file and sets up the number of LPs on each PE */ void codes_mapping_setup_with_seed_offset(int offset) { int grp, lpt, message_size; int pes = tw_nnodes(); lps_per_pe_floor = 0; for (grp = 0; grp < lpconf.lpgroups_count; grp++) { for (lpt = 0; lpt < lpconf.lpgroups[grp].lptypes_count; lpt++) lps_per_pe_floor += (lpconf.lpgroups[grp].lptypes[lpt].count * lpconf.lpgroups[grp].repetitions); } tw_lpid global_nlps = lps_per_pe_floor; lps_leftover = lps_per_pe_floor % pes; lps_per_pe_floor /= pes; //printf("\n LPs for this PE are %d reps %d ", lps_per_pe_floor, lpconf.lpgroups[grp].repetitions); g_tw_mapping=CUSTOM; g_tw_custom_initial_mapping=&codes_mapping_init; g_tw_custom_lp_global_to_local_map=&codes_mapping_to_lp; // configure mem-factor int mem_factor_conf; int rc = configuration_get_value_int(&config, "PARAMS", "pe_mem_factor", NULL, &mem_factor_conf); if (rc == 0 && mem_factor_conf > 0) mem_factor = mem_factor_conf; g_tw_events_per_pe = mem_factor * codes_mapping_get_lps_for_pe(); configuration_get_value_int(&config, "PARAMS", "message_size", NULL, &message_size); if(!message_size) { message_size = 256; printf("\n Warning: ross message size not defined, resetting it to %d", message_size); } // we increment the number of RNGs used to let codes_local_latency use the // last one g_tw_nRNG_per_lp++; tw_define_lps(codes_mapping_get_lps_for_pe(), message_size); // use a similar computation to codes_mapping_init to compute the lpids and // offsets to use in tw_rand_initial_seed // an "offset" of 0 reverts to default RNG seeding behavior - see // ross/rand-clcg4.c for the specific computation // an "offset" < 0 is ignored if (offset > 0){ for (tw_lpid l = 0; l < g_tw_nlp; l++){ for (unsigned int i = 0; i < g_tw_nRNG_per_lp; i++){ tw_rand_initial_seed(&g_tw_lp[l]->rng[i], (g_tw_lp[l]->gid + global_nlps * offset) * g_tw_nRNG_per_lp + i); } } } } void codes_mapping_setup(){ codes_mapping_setup_with_seed_offset(0); } /* given the group and LP type name, return the annotation (or NULL) */ const char* codes_mapping_get_annotation_by_name( const char * group_name, const char * lp_type_name){ for (int g = 0; g < lpconf.lpgroups_count; g++){ const config_lpgroup_t *lpg = &lpconf.lpgroups[g]; if (strcmp(lpg->name.ptr, group_name) == 0){ // group found, iterate through types for (int l = 0; l < lpg->lptypes_count; l++){ const config_lptype_t *lpt = &lpg->lptypes[l]; if (strcmp(lpt->name.ptr, lp_type_name) == 0){ // type found, return the annotation return lpt->anno.ptr; } } } } tw_error(TW_LOC, "unable to find annotation using " "group \"%s\" and lp_type_name \"%s\"", group_name, lp_type_name); return NULL; } const char* codes_mapping_get_annotation_by_lpid(tw_lpid gid){ int group_index, lp_type_index, dummy; codes_mapping_get_lp_info(gid, NULL, &group_index, NULL, &lp_type_index, NULL, &dummy, &dummy); return lpconf.lpgroups[group_index].lptypes[lp_type_index].anno.ptr; } /* * Returns a mapping of LP name to all annotations used with the type * * lp_name - lp name as used in the configuration */ const config_anno_map_t * codes_mapping_get_lp_anno_map(const char *lp_name){ for (int i = 0; i < lpconf.lpannos_count; i++){ if (strcmp(lp_name, lpconf.lpannos[i].lp_name.ptr) == 0){ return &lpconf.lpannos[i]; } } return NULL; } static int get_cid_by_name( char const * name, char const * * names, int num_names) { for (int i = 0; i < num_names; i++) if (strcmp(name, names[i]) == 0) return i; return -1; } static char const * get_name_by_cid( int cid, char const * * names, int num_names) { if (cid < 0 || cid >= num_names) return NULL; else return names[cid]; } int codes_mapping_get_group_cid_by_name(char const * group_name) { return get_cid_by_name(group_name, lpconf.group_names, lpconf.lpgroups_count); } int codes_mapping_get_group_cid_by_lpid(tw_lpid id) { char group_name[MAX_NAME_LENGTH]; int ignore; codes_mapping_get_lp_info(id, group_name, &ignore, NULL, &ignore, NULL, &ignore, &ignore); return codes_mapping_get_group_cid_by_name(group_name); } char const * codes_mapping_get_group_name_by_cid(int cid) { return get_name_by_cid(cid, lpconf.group_names, lpconf.lpgroups_count); } int codes_mapping_get_lp_cid_by_name(char const * lp_type_name) { return get_cid_by_name(lp_type_name, lpconf.lp_names, lpconf.num_uniq_lptypes); } int codes_mapping_get_lp_cid_by_lpid(tw_lpid id) { char lp_name[MAX_NAME_LENGTH]; int ignore; codes_mapping_get_lp_info(id, NULL, &ignore, lp_name, &ignore, NULL, &ignore, &ignore); return codes_mapping_get_lp_cid_by_name(lp_name); } char const * codes_mapping_get_lp_name_by_cid(int cid) { return get_name_by_cid(cid, lpconf.lp_names, lpconf.num_uniq_lptypes); } int codes_mapping_get_anno_cid_by_name(char const * annotation) { if (annotation == NULL || annotation[0] == '\0') return lpconf.num_uniq_annos; else return get_cid_by_name(annotation, lpconf.anno_names, lpconf.num_uniq_annos); } int codes_mapping_get_anno_cid_by_lpid(tw_lpid id) { char anno[MAX_NAME_LENGTH]; int ignore; codes_mapping_get_lp_info(id, NULL, &ignore, NULL, &ignore, anno, &ignore, &ignore); return codes_mapping_get_anno_cid_by_name(anno); } char const * codes_mapping_get_anno_name_by_cid(int cid) { return get_name_by_cid(cid, lpconf.anno_names, lpconf.num_uniq_annos); } /* * Local variables: * c-indent-level: 4 * c-basic-offset: 4 * End: * * vim: ts=8 sts=4 sw=4 expandtab */