diff --git a/codes/model-net-lp.h b/codes/model-net-lp.h index c8f87db66c9e85f57286708556261dfb6381f89e..001d58d8da3c9ae505cb2f688b133b0b73d63699 100644 --- a/codes/model-net-lp.h +++ b/codes/model-net-lp.h @@ -24,6 +24,12 @@ extern int model_net_base_magic; +// register the networks with ROSS, given the array of flags, one for each +// network type +void model_net_base_register(int *do_config_nets); +// configure the base LP type, setting up general parameters +void model_net_base_configure(); + // Construct a model-net-specific event, analagous to a tw_event_new and // codes_event_new. The difference here is that we return pointers to // both the message data (to be cast into the appropriate type) and the @@ -49,7 +55,7 @@ void * model_net_method_get_edata(int net_id, void * msg); /// use them // init method -void model_net_base_init(int id_count, int *ids); +void model_net_base_init(int *do_config_nets); enum model_net_base_event_type { MN_BASE_NEW_MSG, diff --git a/codes/model-net-method.h b/codes/model-net-method.h index 8da93408dbc901200e9ed62f7d992fad48409f5c..18a82718d02906188697b1c6d781ab134ccbf773 100644 --- a/codes/model-net-method.h +++ b/codes/model-net-method.h @@ -12,7 +12,7 @@ struct model_net_method { uint64_t packet_size; /* packet size */ - void (*mn_setup)(const void* net_params); /* For initializing the network */ + void (*mn_configure)(); /* For initializing the network */ tw_stime (*model_net_method_packet_event)( char* category, tw_lpid final_dest_lp, diff --git a/codes/model-net-sched.h b/codes/model-net-sched.h index 71069ab9f626119777ed0ab48cbec0d7ac075c29..82a0e90c277a8818d9bea52ff7a3285c4642b2d9 100644 --- a/codes/model-net-sched.h +++ b/codes/model-net-sched.h @@ -30,10 +30,6 @@ enum sched_type { extern char * sched_names[]; -/// global for scheduler -/// TODO: move away from using the global for when we have multiple networks -extern enum sched_type mn_sched_type; - /// scheduler decls typedef struct model_net_sched_s model_net_sched; diff --git a/codes/model-net.h b/codes/model-net.h index d7486dd040f167183a55584bb5716e44f6c2b898..a5c1da3f4c60ac0027e87cb405b21c0daf4c2d01 100644 --- a/codes/model-net.h +++ b/codes/model-net.h @@ -24,7 +24,8 @@ * messages as part of an event and expects FCFS ordering. A proper fix which * involves model-net LP-level scheduling of requests is ideal, but not * feasible for now (would basically have to redesign model-net), so expose - * explicit start-sequence and stop-sequence markers as a workaround */ + * explicit start-sequence and stop-sequence markers as a workaround + */ extern int in_sequence; extern tw_stime mn_msg_offset; #define MN_START_SEQ() do {\ @@ -36,11 +37,6 @@ extern tw_stime mn_msg_offset; } while (0) -typedef struct simplenet_param simplenet_param; -typedef struct simplewan_param simplewan_param; -typedef struct dragonfly_param dragonfly_param; -typedef struct torus_param torus_param; -typedef struct loggp_param loggp_param; typedef struct mn_stats mn_stats; // use the X-macro to get types and names rolled up into one structure @@ -93,53 +89,18 @@ struct mn_stats long max_event_size; }; -/* structs for initializing a network/ specifying network parameters */ -struct loggp_param -{ - char* net_config_file; /* file with loggp parameter table */ -}; - -/* structs for initializing a network/ specifying network parameters */ -struct simplenet_param -{ - double net_startup_ns; /*simplenet startup cost*/ - double net_bw_mbps; /*Link bandwidth per byte*/ - int num_nics; -}; - -struct simplewan_param -{ - char bw_filename[MAX_NAME_LENGTH]; - char startup_filename[MAX_NAME_LENGTH]; -}; +/* Registers all model-net LPs in ROSS. Should be called after + * configuration_load, but before codes_mapping_setup */ +void model_net_register(); -struct dragonfly_param -{ - int num_routers; /*Number of routers in a group*/ - double local_bandwidth;/* bandwidth of the router-router channels within a group */ - double global_bandwidth;/* bandwidth of the inter-group router connections */ - double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */ - int num_vcs; /* number of virtual channels */ - int local_vc_size; /* buffer size of the router-router channels */ - int global_vc_size; /* buffer size of the global channels */ - int cn_vc_size; /* buffer size of the compute node channels */ - short routing; /* minimal or non-minimal routing */ - short chunk_size; /* full-sized packets are broken into smaller chunks.*/ -}; - -struct torus_param -{ - int n_dims; /*Dimension of the torus network, 5-D, 7-D or any other*/ - int* dim_length; /*Length of each torus dimension*/ - double link_bandwidth;/* bandwidth for each torus link */ - int buffer_size; /* number of buffer slots for each vc in flits*/ - int num_vc; /* number of virtual channels for each torus link */ - float mean_process;/* mean process time for each flit */ - int chunk_size; /* chunk is the smallest unit--default set to 32 */ -}; - /* In general we need to figure out how to pass configuration information to - * the methods and we need to be able to calculate ross event message size. - */ +/* Configures all model-net LPs based on the CODES configuration, and returns + * ids to address the different types by. + * + * id_count - the output number of networks + * + * return - the set of network IDs, indexed in the order given by the + * modelnet_order configuration parameter */ +int* model_net_configure(int *id_count); /* Initialize/configure the network(s) based on the CODES configuration. * returns an array of the network ids, indexed in the order given by the diff --git a/src/models/networks/model-net/dragonfly.c b/src/models/networks/model-net/dragonfly.c index d9a435b5ec037beadd89cbf0d2dff593d42b0721..e7bd1becd5a16541490676e9f5a2482909d58104 100644 --- a/src/models/networks/model-net/dragonfly.c +++ b/src/models/networks/model-net/dragonfly.c @@ -40,34 +40,45 @@ static double maxd(double a, double b) { return a < b ? b : a; } // arrival rate static double MEAN_INTERVAL=200.0; -/* radix of a dragonfly router = number of global channels + number of - * compute node channels + number of local router channels */ -static int radix=0; - -/* configurable parameters, coming from the codes config file*/ -/* number of virtual channels, number of routers comes from the - * config file, number of compute nodes, global channels and group - * is calculated from these configurable parameters */ -static int num_vcs, num_routers, num_cn, num_global_channels, num_groups, chunk_size; - -/* adaptive threshold is to bias the adaptive routing */ -static int total_routers, adaptive_threshold = 10; +// threshold for adaptive routing +static int adaptive_threshold = 10; /* minimal and non-minimal packet counts for adaptive routing*/ int minimal_count=0, nonmin_count=0; -/* configurable parameters, global channel, local channel and - * compute node bandwidth */ -static double global_bandwidth, local_bandwidth, cn_bandwidth; - -/*configurable parameters, global virtual channel size, local - * virtual channel size and compute node channel size */ -static int global_vc_size, local_vc_size, cn_vc_size; +typedef struct dragonfly_param dragonfly_param; +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static uint64_t num_params = 0; +static dragonfly_param * all_params = NULL; +static const config_anno_map_t * anno_map = NULL; /* global variables for codes mapping */ static char lp_group_name[MAX_NAME_LENGTH]; static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset; +struct dragonfly_param +{ + // configuration parameters + int num_routers; /*Number of routers in a group*/ + double local_bandwidth;/* bandwidth of the router-router channels within a group */ + double global_bandwidth;/* bandwidth of the inter-group router connections */ + double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */ + int num_vcs; /* number of virtual channels */ + int local_vc_size; /* buffer size of the router-router channels */ + int global_vc_size; /* buffer size of the global channels */ + int cn_vc_size; /* buffer size of the compute node channels */ + int routing; /* minimal or non-minimal routing */ + int chunk_size; /* full-sized packets are broken into smaller chunks.*/ + + // derived parameters + int num_cn; + int num_groups; + int radix; + int total_routers; + int num_global_channels; +}; + /* handles terminal and router events like packet generate/send/receive/buffer */ typedef enum event_t event_t; @@ -116,6 +127,9 @@ struct terminal_state /* to maintain a count of child nodes that have fanned in at the parent during the collective fan-in phase*/ int num_fan_nodes; + + const char * anno; + const dragonfly_param *params; }; /* terminal event type (1-4) */ @@ -169,13 +183,13 @@ struct router_state tw_stime* next_credit_available_time; int* vc_occupancy; int* output_vc_state; + + const char * anno; + const dragonfly_param *params; }; static short routing = MINIMAL; -static int head_delay; -static uint64_t num_chunks; - static tw_stime dragonfly_total_time = 0; static tw_stime dragonfly_max_latency = 0; static tw_stime max_collective = 0; @@ -193,32 +207,106 @@ static int dragonfly_get_msg_sz(void) return sizeof(terminal_message); } +static void dragonfly_read_config(const char * anno, dragonfly_param *params){ + // shorthand + dragonfly_param *p = params; -/* setup the dragonfly model, initialize global parameters */ -static void dragonfly_setup(const void* net_params) -{ - dragonfly_param* d_param = (dragonfly_param*)net_params; - - num_vcs = d_param->num_vcs; - num_routers = d_param->num_routers; - num_cn = num_routers/2; - num_global_channels = num_routers/2; - num_groups = num_routers * num_cn + 1; - - global_bandwidth = d_param->global_bandwidth; - local_bandwidth = d_param->local_bandwidth; - cn_bandwidth = d_param->cn_bandwidth; - - global_vc_size = d_param->global_vc_size; - local_vc_size = d_param->local_vc_size; - cn_vc_size = d_param->cn_vc_size; - routing = d_param->routing; - chunk_size = d_param->chunk_size; - - radix = num_vcs * (num_cn + num_global_channels + num_routers); - total_routers = num_groups * num_routers; - lp_type_register("dragonfly_router", dragonfly_get_router_lp_type()); - return; + configuration_get_value_int(&config, "PARAMS", "num_routers", anno, + &p->num_routers); + if(p->num_routers <= 0) { + p->num_routers = 4; + fprintf(stderr, "Number of dimensions not specified, setting to %d\n", + p->num_routers); + } + + configuration_get_value_int(&config, "PARAMS", "num_vcs", anno, + &p->num_vcs); + if(p->num_vcs <= 0) { + p->num_vcs = 1; + fprintf(stderr, "Number of virtual channels not specified, setting to %d\n", p->num_vcs); + } + + configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size); + if(!p->local_vc_size) { + p->local_vc_size = 1024; + fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size); + } + + configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size); + if(!p->global_vc_size) { + p->global_vc_size = 2048; + fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size); + } + + configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size); + if(!p->cn_vc_size) { + p->cn_vc_size = 1024; + fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size); + } + + configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size); + if(!p->chunk_size) { + p->chunk_size = 64; + fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size); + } + + configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth); + if(!p->local_bandwidth) { + p->local_bandwidth = 5.25; + fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth); + } + + configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth); + if(!p->global_bandwidth) { + p->global_bandwidth = 4.7; + fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth); + } + + configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth); + if(!p->cn_bandwidth) { + p->cn_bandwidth = 5.25; + fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth); + } + + + char routing[MAX_NAME_LENGTH]; + configuration_get_value(&config, "PARAMS", "routing", anno, routing, + MAX_NAME_LENGTH); + if(strcmp(routing, "minimal") == 0) + p->routing = 0; + else if(strcmp(routing, "nonminimal")==0 || strcmp(routing,"non-minimal")==0) + p->routing = 1; + else if (strcmp(routing, "adaptive") == 0) + p->routing = 2; + else + { + fprintf(stderr, + "No routing protocol specified, setting to minimal routing\n"); + p->routing = 0; + } + + // set the derived parameters + p->num_cn = p->num_routers/2; + p->num_global_channels = p->num_routers/2; + p->num_groups = p->num_routers * p->num_cn + 1; + p->radix = p->num_vcs * + (p->num_cn + p->num_global_channels + p->num_routers); + p->total_routers = p->num_groups * p->num_routers; +} + +static void dragonfly_configure(){ + anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM); + assert(anno_map); + num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0); + all_params = malloc(num_params * sizeof(*all_params)); + + for (uint64_t i = 0; i < anno_map->num_annos; i++){ + const char * anno = anno_map->annotations[i]; + dragonfly_read_config(anno, &all_params[i]); + } + if (anno_map->has_unanno_lp > 0){ + dragonfly_read_config(NULL, &all_params[anno_map->num_annos]); + } } /* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */ @@ -314,15 +402,8 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui tw_event * e_new; tw_stime xfer_to_nic_time; terminal_message * msg; - tw_lpid dest_nic_id; char* tmp_ptr; - //TODO: be annotation-aware - codes_mapping_get_lp_info(final_dest_lp, lp_group_name, &mapping_grp_id, - NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1, - mapping_rep_id, mapping_offset, &dest_nic_id); - xfer_to_nic_time = codes_local_latency(sender); /* Throws an error of found last KP time > current event time otherwise when LPs of one type are placed together*/ //printf("\n transfer in time %f %f ", xfer_to_nic_time+offset, tw_now(sender)); //e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender); @@ -331,7 +412,6 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui sender, DRAGONFLY, (void**)&msg, (void**)&tmp_ptr); strcpy(msg->category, category); msg->final_dest_gid = final_dest_lp; - msg->dest_terminal_id = dest_nic_id; msg->sender_lp=src_lp; msg->packet_size = packet_size; msg->remote_event_size_bytes = 0; @@ -372,17 +452,18 @@ static void dragonfly_packet_event_rc(tw_lp *sender) tw_lpid getRouterFromGroupID(int gid, router_state * r) { - int group_begin = r->group_id * num_routers; - int group_end = (r->group_id * num_routers) + num_routers-1; - int offset = (gid * num_routers - group_begin) / num_routers; + const dragonfly_param *p = r->params; + int group_begin = r->group_id * p->num_routers; + int group_end = (r->group_id * p->num_routers) + p->num_routers-1; + int offset = (gid * p->num_routers - group_begin) / p->num_routers; - if((gid * num_routers) < group_begin) - offset = (group_begin - gid * num_routers) / num_routers; // take absolute value + if((gid * p->num_routers) < group_begin) + offset = (group_begin - gid * p->num_routers) / p->num_routers; // take absolute value - int half_channel = num_global_channels / 2; - int index = (offset - 1)/(half_channel * num_routers); + int half_channel = p->num_global_channels / 2; + int index = (offset - 1)/(half_channel * p->num_routers); - offset=(offset - 1) % (half_channel * num_routers); + offset=(offset - 1) % (half_channel * p->num_routers); // If the destination router is in the same group tw_lpid router_id; @@ -405,30 +486,31 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw int dest=0, credit_delay=0, type = R_BUFFER; int is_terminal = 0; + const dragonfly_param *p = s->params; // Notify sender terminal about available buffer space if(msg->last_hop == TERMINAL) { dest = msg->src_terminal_id; //determine the time in ns to transfer the credit - credit_delay = (1 / cn_bandwidth) * CREDIT_SIZE; + credit_delay = (1 / p->cn_bandwidth) * CREDIT_SIZE; type = T_BUFFER; is_terminal = 1; } else if(msg->last_hop == GLOBAL) { dest = msg->intm_lp_id; - credit_delay = (1 / global_bandwidth) * CREDIT_SIZE; + credit_delay = (1 / p->global_bandwidth) * CREDIT_SIZE; } else if(msg->last_hop == LOCAL) { dest = msg->intm_lp_id; - credit_delay = (1/local_bandwidth) * CREDIT_SIZE; + credit_delay = (1/p->local_bandwidth) * CREDIT_SIZE; } else printf("\n Invalid message type"); // Assume it takes 0.1 ns of serialization latency for processing the credits in the queue - int output_port = msg->saved_vc / num_vcs; + int output_port = msg->saved_vc / p->num_vcs; msg->saved_available_time = s->next_credit_available_time[output_port]; s->next_credit_available_time[output_port] = maxd(tw_now(lp), s->next_credit_available_time[output_port]); ts = credit_delay + 0.1 + tw_rand_exponential(lp->rng, (double)credit_delay/1000); @@ -456,11 +538,22 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw /* generates packet at the current dragonfly compute node */ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) { + tw_lpid dest_terminal_id; + codes_mapping_get_lp_info(msg->final_dest_gid, lp_group_name, &mapping_grp_id, + NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, s->anno, 0, + mapping_rep_id, mapping_offset, &dest_terminal_id); + msg->dest_terminal_id = dest_terminal_id; + + const dragonfly_param *p = s->params; + tw_stime ts; tw_event *e; terminal_message *m; int i, total_event_size; - num_chunks = msg->packet_size / chunk_size; + uint64_t num_chunks = msg->packet_size / p->chunk_size; + if (msg->packet_size % s->params->chunk_size) + num_chunks++; msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter + tw_rand_integer(lp->rng, 0, lp->gid + g_tw_nlp * s->packet_counter); msg->travel_start_time = tw_now(lp); msg->my_N_hop = 0; @@ -470,9 +563,9 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ // msg->my_N_hop = 0; generating a packet, check if the input queue is available ts = g_tw_lookahead + 0.1 + tw_rand_exponential(lp->rng, MEAN_INTERVAL/200); int chan = -1, j; - for(j = 0; j < num_vcs; j++) + for(j = 0; j < p->num_vcs; j++) { - if(s->vc_occupancy[j] < cn_vc_size * num_chunks) + if(s->vc_occupancy[j] < p->cn_vc_size * num_chunks) { chan=j; break; @@ -487,6 +580,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ e = model_net_method_event_new(lp->gid, i+ts, lp, DRAGONFLY, (void**)&m, &m_data); memcpy(m, msg, sizeof(terminal_message)); + m->dest_terminal_id = dest_terminal_id; void * m_data_src = model_net_method_get_edata(DRAGONFLY, msg); if (msg->remote_event_size_bytes){ memcpy(m_data, m_data_src, msg->remote_event_size_bytes); @@ -523,7 +617,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); stat->send_count++; stat->send_bytes += msg->packet_size; - stat->send_time += (1/cn_bandwidth) * msg->packet_size; + stat->send_time += (1/p->cn_bandwidth) * msg->packet_size; if(stat->max_event_size < total_event_size) stat->max_event_size = total_event_size; return; @@ -541,7 +635,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * // Each packet is broken into chunks and then sent over the channel msg->saved_available_time = s->terminal_available_time; - head_delay = (1/cn_bandwidth) * chunk_size; + double head_delay = (1/s->params->cn_bandwidth) * s->params->chunk_size; ts = head_delay + tw_rand_exponential(lp->rng, (double)head_delay/200); s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp)); s->terminal_available_time += ts; @@ -554,6 +648,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * // we are sending an event to the router, so no method_event here e = tw_event_new(router_id, s->terminal_available_time - tw_now(lp), lp); + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; + if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) printf("\n terminal %d packet %lld chunk %d being sent to router %d router id %d ", (int)lp->gid, (long long)msg->packet_ID, msg->chunk_id, (int)router_id, s->router_id); m = tw_event_data(e); @@ -581,7 +679,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * void* local_event = (char*)model_net_method_get_edata(DRAGONFLY, msg) + msg->remote_event_size_bytes; - ts = g_tw_lookahead + (1/cn_bandwidth) * msg->local_event_size_bytes; + ts = g_tw_lookahead + (1/s->params->cn_bandwidth) * msg->local_event_size_bytes; e_new = tw_event_new(msg->sender_lp, ts, lp); m_new = tw_event_data(e_new); memcpy(m_new, local_event, msg->local_event_size_bytes); @@ -592,7 +690,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * s->packet_counter++; s->vc_occupancy[vc]++; - if(s->vc_occupancy[vc] >= (cn_vc_size * num_chunks)) + if(s->vc_occupancy[vc] >= (s->params->cn_vc_size * num_chunks)) s->output_vc_state[vc] = VC_CREDIT; return; } @@ -600,6 +698,9 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * /* packet arrives at the destination terminal */ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) { + uint64_t num_chunks = msg->packet_size / s->params->chunk_size; + if (msg->packet_size % s->params->chunk_size) + num_chunks++; #if DEBUG if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) { @@ -644,7 +745,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) if(msg->remote_event_size_bytes) { void * tmp_ptr = model_net_method_get_edata(DRAGONFLY, msg); - ts = g_tw_lookahead + 0.1 + (1/cn_bandwidth) * msg->remote_event_size_bytes; + ts = g_tw_lookahead + 0.1 + (1/s->params->cn_bandwidth) * msg->remote_event_size_bytes; if (msg->is_pull){ int net_id = model_net_get_id(LP_METHOD_NM); model_net_event(net_id, msg->category, msg->sender_lp, @@ -660,7 +761,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) } } - int credit_delay = (1 / cn_bandwidth) * CREDIT_SIZE; + int credit_delay = (1 / s->params->cn_bandwidth) * CREDIT_SIZE; ts = credit_delay + 0.1 + tw_rand_exponential(lp->rng, credit_delay/1000); msg->saved_credit_time = s->next_credit_available_time; @@ -671,7 +772,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) //TODO: be annotation-aware codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); - codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", NULL, 1, + codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", s->anno, 0, s->router_id, 0, &router_dest_id); // no method_event here - message going to router buf_e = tw_event_new(router_dest_id, s->next_credit_available_time - tw_now(lp), lp); @@ -691,21 +792,34 @@ terminal_init( terminal_state * s, tw_lp * lp ) { int i; + char anno[MAX_NAME_LENGTH]; + // Assign the global router ID // TODO: be annotation-aware - codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); + codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, + &mapping_type_id, anno, &mapping_rep_id, &mapping_offset); + if (anno[0] == '\0'){ + s->anno = NULL; + s->params = &all_params[num_params-1]; + } + else{ + s->anno = anno; + int id = configuration_get_annotation_index(anno, anno_map); + s->params = &all_params[id]; + } + int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM, - NULL, 1); + s->anno, 0); s->terminal_id = (mapping_rep_id * num_lps) + mapping_offset; - s->router_id=(int)s->terminal_id / num_routers; + s->router_id=(int)s->terminal_id / s->params->num_routers; s->terminal_available_time = 0.0; s->packet_counter = 0; - s->vc_occupancy = (int*)malloc(num_vcs * sizeof(int)); - s->output_vc_state = (int*)malloc(num_vcs * sizeof(int)); + s->vc_occupancy = (int*)malloc(s->params->num_vcs * sizeof(int)); + s->output_vc_state = (int*)malloc(s->params->num_vcs * sizeof(int)); - for( i = 0; i < num_vcs; i++ ) + for( i = 0; i < s->params->num_vcs; i++ ) { s->vc_occupancy[i]=0; s->output_vc_state[i]=VC_IDLE; @@ -768,7 +882,7 @@ static void send_remote_event(terminal_state * s, tw_event* e; tw_stime ts; terminal_message * m; - ts = (1/cn_bandwidth) * msg->remote_event_size_bytes; + ts = (1/s->params->cn_bandwidth) * msg->remote_event_size_bytes; e = codes_event_new(s->origin_svr, ts, lp); m = tw_event_data(e); char* tmp_ptr = (char*)msg; @@ -802,8 +916,8 @@ static void node_collective_init(terminal_state * s, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM, - NULL, 1); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1, + s->anno, 0); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, s->anno, 0, s->parent_node_id/num_lps, (s->parent_node_id % num_lps), &parent_nic_id); @@ -839,7 +953,7 @@ static void node_collective_fan_in(terminal_state * s, codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM, - NULL, 1); + s->anno, 0); tw_event* e_new; terminal_message * msg_new; @@ -858,7 +972,7 @@ static void node_collective_fan_in(terminal_state * s, xfer_to_nic_time = g_tw_lookahead + LEVEL_DELAY; /* get the global LP ID of the parent node */ - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1, + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, s->anno, 0, s->parent_node_id/num_lps, (s->parent_node_id % num_lps), &parent_nic_id); @@ -952,7 +1066,7 @@ static void node_collective_fan_out(terminal_state * s, /* get global LP ID of the child node */ codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, - NULL, 1, s->children[i]/num_lps, + s->anno, 0, s->children[i]/num_lps, (s->children[i] % num_lps), &child_nic_id); //e_new = codes_event_new(child_nic_id, xfer_to_nic_time, lp); //msg_new = tw_event_data(e_new); @@ -1071,8 +1185,8 @@ get_next_stop(router_state * s, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM, - NULL, 1); - int dest_router_id = (mapping_offset + (mapping_rep_id * num_lps)) / num_routers; + s->anno, 0); + int dest_router_id = (mapping_offset + (mapping_rep_id * num_lps)) / s->params->num_routers; codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); @@ -1090,10 +1204,10 @@ get_next_stop(router_state * s, /* Generate inter-mediate destination for non-minimal routing (selecting a random group) */ if(msg->last_hop == TERMINAL && msg->path_type == NON_MINIMAL) { - if(dest_router_id / num_routers != s->group_id) + if(dest_router_id / s->params->num_routers != s->group_id) { bf->c2 = 1; - int intm_grp_id = tw_rand_integer(lp->rng, 0, num_groups-1); + int intm_grp_id = tw_rand_integer(lp->rng, 0, s->params->num_groups-1); //int intm_grp_id = (s->group_id + s->group_id/2) % num_groups; msg->intm_group_id = intm_grp_id; } @@ -1110,7 +1224,7 @@ get_next_stop(router_state * s, } else /* direct the packet to the destination group */ { - dest_group_id = dest_router_id / num_routers; + dest_group_id = dest_router_id / s->params->num_routers; } /* It means the packet has arrived at the destination group. Now divert it to the destination router. */ @@ -1125,14 +1239,14 @@ get_next_stop(router_state * s, if(dest_lp == local_router_id) { - for(i=0; i < num_global_channels; i++) + for(i=0; i < s->params->num_global_channels; i++) { - if(s->global_channel[i] / num_routers == dest_group_id) + if(s->global_channel[i] / s->params->num_routers == dest_group_id) dest_lp=s->global_channel[i]; } } } - codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", NULL, 1, dest_lp, + codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", s->anno, 0, dest_lp, 0, &router_dest_id); return router_dest_id; } @@ -1149,12 +1263,13 @@ get_output_port( router_state * s, codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); - int num_lps = codes_mapping_get_lp_count(lp_group_name,1,LP_CONFIG_NM,NULL,1); + int num_lps = codes_mapping_get_lp_count(lp_group_name,1,LP_CONFIG_NM,s->anno,0); terminal_id = (mapping_rep_id * num_lps) + mapping_offset; if(next_stop == msg->dest_terminal_id) { - output_port = num_routers + num_global_channels + ( terminal_id % num_cn); + output_port = s->params->num_routers + s->params->num_global_channels + + ( terminal_id % s->params->num_cn); //if(output_port > 6) // printf("\n incorrect output port %d terminal id %d ", output_port, terminal_id); } @@ -1163,19 +1278,19 @@ get_output_port( router_state * s, codes_mapping_get_lp_info(next_stop, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); int local_router_id = mapping_rep_id + mapping_offset; - int intm_grp_id = local_router_id / num_routers; + int intm_grp_id = local_router_id / s->params->num_routers; if(intm_grp_id != s->group_id) { - for(i=0; i < num_global_channels; i++) + for(i=0; i < s->params->num_global_channels; i++) { if(s->global_channel[i] == local_router_id) - output_port = num_routers + i; + output_port = s->params->num_routers + i; } } else { - output_port = local_router_id % num_routers; + output_port = local_router_id % s->params->num_routers; } // printf("\n output port not found %d next stop %d local router id %d group id %d intm grp id %d %d", output_port, next_stop, local_router_id, s->group_id, intm_grp_id, local_router_id%num_routers); } @@ -1194,12 +1309,17 @@ router_packet_send( router_state * s, terminal_message *m; int next_stop = -1, output_port = -1, output_chan = -1; - float bandwidth = local_bandwidth; - int path = routing; + float bandwidth = s->params->local_bandwidth; + int path = s->params->routing; int minimal_out_port = -1, nonmin_out_port = -1; bf->c3 = 0; - if(msg->last_hop == TERMINAL && routing == ADAPTIVE) + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; + + + if(msg->last_hop == TERMINAL && s->params->routing == ADAPTIVE) { // decide which routing to take int minimal_next_stop=get_next_stop(s, bf, msg, lp, MINIMAL); @@ -1208,8 +1328,8 @@ router_packet_send( router_state * s, nonmin_out_port = get_output_port(s, bf, msg, lp, nonmin_next_stop); int nonmin_port_count = s->vc_occupancy[nonmin_out_port]; int min_port_count = s->vc_occupancy[minimal_out_port]; - int nonmin_vc = s->vc_occupancy[nonmin_out_port * num_vcs + 2]; - int min_vc = s->vc_occupancy[minimal_out_port * num_vcs + 1]; + int nonmin_vc = s->vc_occupancy[nonmin_out_port * s->params->num_vcs + 2]; + int min_vc = s->vc_occupancy[minimal_out_port * s->params->num_vcs + 1]; // Adaptive routing condition from the dragonfly paper Page 83 // modified according to booksim adaptive routing condition @@ -1242,7 +1362,7 @@ router_packet_send( router_state * s, next_stop = get_next_stop(s, bf, msg, lp, path); output_port = get_output_port(s, bf, msg, lp, next_stop); } - output_chan = output_port * num_vcs; + output_chan = output_port * s->params->num_vcs; // Even numbered channels for minimal routing // Odd numbered channels for nonminimal routing @@ -1258,24 +1378,25 @@ router_packet_send( router_state * s, }*/ int global=0; - int buf_size = local_vc_size; + int buf_size = s->params->local_vc_size; assert(output_port != -1); assert(output_chan != -1); // Allocate output Virtual Channel - if(output_port >= num_routers && output_port < num_routers + num_global_channels) + if(output_port >= s->params->num_routers && + output_port < s->params->num_routers + s->params->num_global_channels) { - bandwidth = global_bandwidth; + bandwidth = s->params->global_bandwidth; global = 1; - buf_size = global_vc_size; + buf_size = s->params->global_vc_size; } - if(output_port >= num_routers + num_global_channels) - buf_size = cn_vc_size; + if(output_port >= s->params->num_routers + s->params->num_global_channels) + buf_size = s->params->cn_vc_size; if(s->vc_occupancy[output_chan] >= buf_size) { - printf("\n %lf Router %ld buffers overflowed from incoming terminals channel %d occupancy %d radix %d next_stop %d ", tw_now(lp),(long int) lp->gid, output_chan, s->vc_occupancy[output_chan], radix, next_stop); + printf("\n %lf Router %ld buffers overflowed from incoming terminals channel %d occupancy %d radix %d next_stop %d ", tw_now(lp),(long int) lp->gid, output_chan, s->vc_occupancy[output_chan], s->params->radix, next_stop); bf->c3 = 1; return; //MPI_Finalize(); @@ -1292,7 +1413,7 @@ if( msg->packet_ID == TRACK && next_stop != msg->dest_terminal_id && msg->chunk_ #endif // If source router doesn't have global channel and buffer space is available, then assign to appropriate intra-group virtual channel msg->saved_available_time = s->next_output_available_time[output_port]; - ts = g_tw_lookahead + 0.1 + ((1/bandwidth) * chunk_size) + tw_rand_exponential(lp->rng, (double)chunk_size/200); + ts = g_tw_lookahead + 0.1 + ((1/bandwidth) * s->params->chunk_size) + tw_rand_exponential(lp->rng, (double)s->params->chunk_size/200); s->next_output_available_time[output_port] = maxd(s->next_output_available_time[output_port], tw_now(lp)); s->next_output_available_time[output_port] += ts; @@ -1329,7 +1450,7 @@ if( msg->packet_ID == TRACK && next_stop != msg->dest_terminal_id && msg->chunk_ { m->type = T_ARRIVE; - if(s->vc_occupancy[output_chan] >= cn_vc_size * num_chunks) + if(s->vc_occupancy[output_chan] >= s->params->cn_vc_size * num_chunks) s->output_vc_state[output_chan] = VC_CREDIT; } else @@ -1340,13 +1461,13 @@ if( msg->packet_ID == TRACK && next_stop != msg->dest_terminal_id && msg->chunk_ /* If this is a global channel then the buffer space is different */ if( global ) { - if(s->vc_occupancy[output_chan] >= global_vc_size * num_chunks ) + if(s->vc_occupancy[output_chan] >= s->params->global_vc_size * num_chunks ) s->output_vc_state[output_chan] = VC_CREDIT; } else { /* buffer space is less for local channels */ - if( s->vc_occupancy[output_chan] >= local_vc_size * num_chunks ) + if( s->vc_occupancy[output_chan] >= s->params->local_vc_size * num_chunks ) s->output_vc_state[output_chan] = VC_CREDIT; } } @@ -1367,7 +1488,9 @@ router_packet_receive( router_state * s, msg->my_N_hop++; ts = g_tw_lookahead + 0.1 + tw_rand_exponential(lp->rng, (double)MEAN_INTERVAL/200); - num_chunks = msg->packet_size/chunk_size; + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) printf("\n packet %lld chunk %d received at router %d ", msg->packet_ID, msg->chunk_id, (int)lp->gid); @@ -1385,21 +1508,36 @@ router_packet_receive( router_state * s, /* sets up the router virtual channels, global channels, local channels, compute node channels */ void router_setup(router_state * r, tw_lp * lp) { - codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, - &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); + char anno[MAX_NAME_LENGTH]; + codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, + &mapping_type_id, anno, &mapping_rep_id, &mapping_offset); + + if (anno[0] == '\0'){ + r->anno = NULL; + r->params = &all_params[num_params-1]; + } + else{ + r->anno = anno; + int id = configuration_get_annotation_index(anno, anno_map); + r->params = &all_params[id]; + } + + // shorthand + const dragonfly_param *p = r->params; + r->router_id=mapping_rep_id + mapping_offset; - r->group_id=r->router_id/num_routers; + r->group_id=r->router_id/p->num_routers; int i; - int router_offset=(r->router_id % num_routers) * (num_global_channels / 2) + 1; + int router_offset=(r->router_id % p->num_routers) * (p->num_global_channels / 2) + 1; - r->global_channel = (int*)malloc(num_global_channels * sizeof(int)); - r->next_output_available_time = (tw_stime*)malloc(radix * sizeof(tw_stime)); - r->next_credit_available_time = (tw_stime*)malloc(radix * sizeof(tw_stime)); - r->vc_occupancy = (int*)malloc(radix * sizeof(int)); - r->output_vc_state = (int*)malloc(radix * sizeof(int)); + r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int)); + r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime)); + r->next_credit_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime)); + r->vc_occupancy = (int*)malloc(p->radix * sizeof(int)); + r->output_vc_state = (int*)malloc(p->radix * sizeof(int)); - for(i=0; i < radix; i++) + for(i=0; i < p->radix; i++) { // Set credit & router occupancy r->next_output_available_time[i]=0; @@ -1409,20 +1547,20 @@ void router_setup(router_state * r, tw_lp * lp) } //round the number of global channels to the nearest even number - for(i=0; i < num_global_channels; i++) + for(i=0; i < p->num_global_channels; i++) { if(i % 2 != 0) { - r->global_channel[i]=(r->router_id + (router_offset * num_routers))%total_routers; + r->global_channel[i]=(r->router_id + (router_offset * p->num_routers))%p->total_routers; router_offset++; } else { - r->global_channel[i]=r->router_id - ((router_offset) * num_routers); + r->global_channel[i]=r->router_id - ((router_offset) * p->num_routers); } if(r->global_channel[i]<0) { - r->global_channel[i]=total_routers+r->global_channel[i]; + r->global_channel[i]=p->total_routers+r->global_channel[i]; } } return; @@ -1463,6 +1601,9 @@ void router_event(router_state * s, tw_bf * bf, terminal_message * msg, tw_lp * /* Reverse computation handler for a terminal event */ void terminal_rc_event_handler(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) { + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; switch(msg->type) { case T_GENERATE: @@ -1471,19 +1612,19 @@ void terminal_rc_event_handler(terminal_state * s, tw_bf * bf, terminal_message tw_rand_reverse_unif(lp->rng); for(i = 0; i < num_chunks; i++) - tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); mn_stats* stat; stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); stat->send_count--; stat->send_bytes -= msg->packet_size; - stat->send_time -= (1/cn_bandwidth) * msg->packet_size; + stat->send_time -= (1/s->params->cn_bandwidth) * msg->packet_size; } break; case T_SEND: { s->terminal_available_time = msg->saved_available_time; - tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); int vc = msg->saved_vc; s->vc_occupancy[vc]--; s->packet_counter--; @@ -1517,8 +1658,7 @@ void terminal_rc_event_handler(terminal_state * s, tw_bf * bf, terminal_message { int msg_indx = msg->vc_index; s->vc_occupancy[msg_indx]++; - - if(s->vc_occupancy[msg_indx] == cn_vc_size * num_chunks) + if(s->vc_occupancy[msg_indx] == s->params->cn_vc_size) s->output_vc_state[msg_indx] = VC_CREDIT; } break; @@ -1561,6 +1701,9 @@ void terminal_rc_event_handler(terminal_state * s, tw_bf * bf, terminal_message /* Reverse computation handler for a router event */ void router_rc_event_handler(router_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) { + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; switch(msg->type) { case R_SEND: @@ -1578,7 +1721,7 @@ void router_rc_event_handler(router_state * s, tw_bf * bf, terminal_message * ms tw_rand_reverse_unif(lp->rng); int output_chan = msg->old_vc; - int output_port = output_chan / num_vcs; + int output_port = output_chan / s->params->num_vcs; s->next_output_available_time[output_port] = msg->saved_available_time; s->vc_occupancy[output_chan]--; @@ -1592,7 +1735,7 @@ void router_rc_event_handler(router_state * s, tw_bf * bf, terminal_message * ms msg->my_N_hop--; tw_rand_reverse_unif(lp->rng); tw_rand_reverse_unif(lp->rng); - int output_port = msg->saved_vc/num_vcs; + int output_port = msg->saved_vc/s->params->num_vcs; s->next_credit_available_time[output_port] = msg->saved_available_time; if (msg->chunk_id == num_chunks-1 && msg->remote_event_size_bytes && @@ -1608,12 +1751,12 @@ void router_rc_event_handler(router_state * s, tw_bf * bf, terminal_message * ms int msg_indx = msg->vc_index; s->vc_occupancy[msg_indx]++; - int buf = local_vc_size; + int buf = s->params->local_vc_size; if(msg->last_hop == GLOBAL) - buf = global_vc_size; + buf = s->params->global_vc_size; else if(msg->last_hop == TERMINAL) - buf = cn_vc_size; + buf = s->params->cn_vc_size; if(s->vc_occupancy[msg_indx] >= buf * num_chunks) s->output_vc_state[msg_indx] = VC_CREDIT; @@ -1672,7 +1815,7 @@ static tw_lpid dragonfly_find_local_device(tw_lp *sender) /* data structure for dragonfly statistics */ struct model_net_method dragonfly_method = { - .mn_setup = dragonfly_setup, + .mn_configure = dragonfly_configure, .model_net_method_packet_event = dragonfly_packet_event, .model_net_method_packet_event_rc = dragonfly_packet_event_rc, .mn_get_lp_type = dragonfly_get_cn_lp_type, diff --git a/src/models/networks/model-net/loggp.c b/src/models/networks/model-net/loggp.c index efcb7e40c0b1f0f1945f6dd2df6bd720217bc87f..ad41dbfb166e659b83e0a440d540f2b03e67089b 100644 --- a/src/models/networks/model-net/loggp.c +++ b/src/models/networks/model-net/loggp.c @@ -26,15 +26,6 @@ /*Define loggp data types and structs*/ typedef struct loggp_state loggp_state; - -struct loggp_state -{ - /* next idle times for network card, both inbound and outbound */ - tw_stime net_send_next_idle; - tw_stime net_recv_next_idle; - struct mn_stats loggp_stats_array[CATEGORY_MAX]; -}; - /* loggp parameters for a given msg size, as reported by netgauge */ struct param_table_entry { @@ -50,8 +41,32 @@ struct param_table_entry double G; double lsqu_gG; }; -struct param_table_entry param_table[100]; -static int param_table_size = 0; +typedef struct param_table_entry param_table_entry; + +typedef struct loggp_param loggp_param; +// loggp parameters +struct loggp_param +{ + int table_size; + param_table_entry table[100]; +}; + + +struct loggp_state +{ + /* next idle times for network card, both inbound and outbound */ + tw_stime net_send_next_idle; + tw_stime net_recv_next_idle; + const char * anno; + const loggp_param *params; + struct mn_stats loggp_stats_array[CATEGORY_MAX]; +}; + +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static uint64_t num_params = 0; +static loggp_param * all_params = NULL; +static const config_anno_map_t * anno_map = NULL; static int loggp_magic = 0; @@ -73,21 +88,11 @@ static void loggp_collective(); /* collective network calls-- rc */ static void loggp_collective_rc(); -/* allocate a new event that will pass through loggp to arriave at its - * destination: - * - * - category: category name to associate with this communication - * - final_dest_gid: the LP that the message should be delivered to. - * - event_size_bytes: size of event msg that will be delivered to - * final_dest_gid. - * - local_event_size_byte: size of event message that will delivered to - * local LP upon local send comletion (set to 0 if not used) - * - net_msg_size_bytes: size of simulated network message in bytes. - * - sender: LP calling this function. - */ /* Modelnet interface events */ /* sets up the loggp parameters through modelnet interface */ -static void loggp_setup(const void* net_params); +static void loggp_configure(); + +static void loggp_set_params(const char * config_file, loggp_param * params); /* Issues a loggp packet event call */ static tw_stime loggp_packet_event( @@ -112,12 +117,14 @@ static void loggp_report_stats(); static tw_lpid loggp_find_local_device(tw_lp *sender); -static struct param_table_entry* find_params(uint64_t msg_size); +static const struct param_table_entry* find_params( + uint64_t msg_size, + const loggp_param *params); /* data structure for model-net statistics */ struct model_net_method loggp_method = { - .mn_setup = loggp_setup, + .mn_configure = loggp_configure, .model_net_method_packet_event = loggp_packet_event, .model_net_method_packet_event_rc = loggp_packet_event_rc, .mn_get_lp_type = loggp_get_lp_type, @@ -220,6 +227,14 @@ static void loggp_init( ns->net_send_next_idle = tw_now(lp); ns->net_recv_next_idle = tw_now(lp); + ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid); + if (ns->anno == NULL) + ns->params = &all_params[num_params-1]; + else{ + int id = configuration_get_annotation_index(ns->anno, anno_map); + ns->params = &all_params[id]; + } + bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2); loggp_magic = h1+h2; /* printf("\n loggp_magic %d ", loggp_magic); */ @@ -324,9 +339,9 @@ static void handle_msg_ready_event( loggp_message *m_new; struct mn_stats* stat; double recv_time; - struct param_table_entry *param; + const struct param_table_entry *param; - param = find_params(m->net_msg_size_bytes); + param = find_params(m->net_msg_size_bytes, ns->params); recv_time = ((double)(m->net_msg_size_bytes-1)*param->G); /* scale to nanoseconds */ @@ -420,9 +435,9 @@ static void handle_msg_start_event( char lp_group_name[MAX_NAME_LENGTH]; int total_event_size; double xmit_time; - struct param_table_entry *param; + const struct param_table_entry *param; - param = find_params(m->net_msg_size_bytes); + param = find_params(m->net_msg_size_bytes, ns->params); total_event_size = model_net_get_msg_sz(LOGGP) + m->event_size_bytes + m->local_event_size_bytes; @@ -465,7 +480,7 @@ static void handle_msg_start_event( // TODO: make annotation-aware codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1, + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0, mapping_rep_id, mapping_offset, &dest_id); void *m_data; @@ -574,52 +589,76 @@ static tw_stime loggp_packet_event( return xfer_to_nic_time; } -static void loggp_setup(const void* net_params) -{ - loggp_param* loggp_params = (loggp_param*)net_params; +static void loggp_configure(){ + char config_file[MAX_NAME_LENGTH]; + + anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM); + assert(anno_map); + num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0); + all_params = malloc(num_params * sizeof(*all_params)); + + for (uint64_t i = 0; i < anno_map->num_annos; i++){ + const char * anno = anno_map->annotations[i]; + int rc = configuration_get_value_relpath(&config, "PARAMS", + "net_config_file", anno, config_file, MAX_NAME_LENGTH); + if (rc == 0){ + tw_error(TW_LOC, "unable to read PARAMS:net_config_file@%s", + anno); + } + loggp_set_params(config_file, &all_params[i]); + } + if (anno_map->has_unanno_lp > 0){ + int rc = configuration_get_value_relpath(&config, "PARAMS", + "net_config_file", NULL, config_file, MAX_NAME_LENGTH); + if (rc == 0){ + tw_error(TW_LOC, "unable to read PARAMS:net_config_file"); + } + loggp_set_params(config_file, &all_params[anno_map->num_annos]); + } +} + +void loggp_set_params(const char * config_file, loggp_param * params){ FILE *conf; int ret; char buffer[512]; int line_nr = 0; + printf("Loggp configured to use parameters from file %s\n", config_file); - /* TODO: update this to read on one proc and then broadcast */ - - printf("Loggp configured to use parameters from file %s\n", loggp_params->net_config_file); - - conf = fopen(loggp_params->net_config_file, "r"); + conf = fopen(config_file, "r"); if(!conf) { perror("fopen"); assert(0); } + params->table_size = 0; while(fgets(buffer, 512, conf)) { line_nr++; if(buffer[0] == '#') continue; ret = sscanf(buffer, "%llu %d %lf %lf %lf %lf %lf %lf %lf %lf %lf", - ¶m_table[param_table_size].size, - ¶m_table[param_table_size].n, - ¶m_table[param_table_size].PRTT_10s, - ¶m_table[param_table_size].PRTT_n0s, - ¶m_table[param_table_size].PRTT_nPRTT_10ss, - ¶m_table[param_table_size].L, - ¶m_table[param_table_size].o_s, - ¶m_table[param_table_size].o_r, - ¶m_table[param_table_size].g, - ¶m_table[param_table_size].G, - ¶m_table[param_table_size].lsqu_gG); + ¶ms->table[params->table_size].size, + ¶ms->table[params->table_size].n, + ¶ms->table[params->table_size].PRTT_10s, + ¶ms->table[params->table_size].PRTT_n0s, + ¶ms->table[params->table_size].PRTT_nPRTT_10ss, + ¶ms->table[params->table_size].L, + ¶ms->table[params->table_size].o_s, + ¶ms->table[params->table_size].o_r, + ¶ms->table[params->table_size].g, + ¶ms->table[params->table_size].G, + ¶ms->table[params->table_size].lsqu_gG); if(ret != 11) { fprintf(stderr, "Error: malformed line %d in %s\n", line_nr, - loggp_params->net_config_file); + config_file); assert(0); } - param_table_size++; + params->table_size++; } - printf("Parsed %d loggp table entries.\n", param_table_size); + printf("Parsed %d loggp table entries.\n", params->table_size); fclose(conf); @@ -634,25 +673,26 @@ static void loggp_packet_event_rc(tw_lp *sender) /* find the parameters corresponding to the message size we are transmitting */ -static struct param_table_entry* find_params(uint64_t msg_size) -{ +static const struct param_table_entry* find_params( + uint64_t msg_size, + const loggp_param *params) { int i; /* pick parameters based on the next smallest size in the table, but * default to beginning or end of table if we are out of range */ - for(i=0; itable_size; i++) { - if(param_table[i].size > msg_size) + if(params->table[i].size > msg_size) { break; } } - if(i>=param_table_size) - i = param_table_size-1; + if(i>=params->table_size) + i = params->table_size-1; - return(¶m_table[i]); + return(¶ms->table[i]); } static tw_lpid loggp_find_local_device(tw_lp *sender) diff --git a/src/models/networks/model-net/model-net-lp.c b/src/models/networks/model-net/model-net-lp.c index 1b14b03f4c973c3febbe67cd4f6024ed2b2d8542..e3fa116e5f739093b517f91ec1f54fb4824b8788 100644 --- a/src/models/networks/model-net/model-net-lp.c +++ b/src/models/networks/model-net/model-net-lp.c @@ -23,18 +23,32 @@ int model_net_base_magic; // issues... static int msg_offsets[MAX_NETS]; +typedef struct model_net_base_params { + enum sched_type stype; + uint64_t packet_size; +} model_net_base_params; + +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static int num_params = 0; +static const char * annos[CONFIGURATION_MAX_ANNOS]; +static model_net_base_params all_params[CONFIGURATION_MAX_ANNOS]; + typedef struct model_net_base_state { int net_id; // whether scheduler loop is running int in_sched_loop; // model-net scheduler model_net_sched *sched; + // parameters + const model_net_base_params * params; // lp type and state of underlying model net method - cache here so we // don't have to constantly look up const tw_lptype *sub_type; void *sub_state; } model_net_base_state; + /**** END SIMULATION DATA STRUCTURES ****/ /**** BEGIN LP, EVENT PROCESSING FUNCTION DECLS ****/ @@ -93,44 +107,132 @@ tw_lptype model_net_base_lp = { /**** BEGIN IMPLEMENTATIONS ****/ -void model_net_base_init(int id_count, int *ids){ +void model_net_base_register(int *do_config_nets){ + // here, we initialize ALL lp types to use the base type + for (int i = 0; i < MAX_NETS; i++){ + if (do_config_nets[i]){ + lp_type_register(model_net_lp_config_names[i], &model_net_base_lp); + // HACK: unfortunately, we need to special-case dragonfly + // registration at the moment - there are two LPs, and + // previously the LP matched on configuration initialized + // the "router" LP. Now that the base LP is in charge of + // registration, we need to take care of it + // TODO: fix the interface to have underlying networks do + // the registration + if (i==DRAGONFLY){ + lp_type_register("dragonfly_router", + &method_array[DRAGONFLY]->mn_get_lp_type()[1]); + } + } + } +} + +static void base_read_config(const char * anno, model_net_base_params *p){ + char sched[MAX_NAME_LENGTH]; + long int packet_size_l = 0; + uint64_t packet_size; + int ret; + + // TODO: make this annotation-specific - put in the config loop, make part + // of model-net base + ret = configuration_get_value(&config, "PARAMS", "modelnet_scheduler", + anno, sched, MAX_NAME_LENGTH); + configuration_get_value_longint(&config, "PARAMS", "packet_size", anno, + &packet_size_l); + packet_size = packet_size_l; + + if (ret > 0){ + int i; + for (i = 0; i < MAX_SCHEDS; i++){ + if (strcmp(sched_names[i], sched) == 0){ + p->stype = i; + break; + } + } + if (i == MAX_SCHEDS){ + tw_error(TW_LOC,"Unknown value for PARAMS:modelnet-scheduler : " + "%s\n", sched); + } + } + else{ + // default: FCFS + p->stype = MN_SCHED_FCFS; + } + + if (p->stype == MN_SCHED_FCFS_FULL){ + // override packet size to something huge (leave a bit in the unlikely + // case that an op using packet size causes overflow) + packet_size = 1ull << 62; + } + else if (!packet_size && p->stype != MN_SCHED_FCFS_FULL) + { + packet_size = 512; + fprintf(stderr, "Warning, no packet size specified, setting packet size to %llu\n", packet_size); + } + + p->packet_size = packet_size; +} + +void model_net_base_configure(){ uint32_t h1=0, h2=0; bj_hashlittle2(MN_NAME, strlen(MN_NAME), &h1, &h2); model_net_base_magic = h1+h2; - // here, we initialize ALL lp types to use the base type - // TODO: only initialize ones that are actually used - for (int i = 0; i < id_count; i++){ - lp_type_register(model_net_lp_config_names[ids[i]], &model_net_base_lp); + // set up offsets - doesn't matter if they are actually used or not + msg_offsets[SIMPLENET] = + offsetof(model_net_wrap_msg, msg.m_snet); + msg_offsets[SIMPLEWAN] = + offsetof(model_net_wrap_msg, msg.m_swan); + msg_offsets[TORUS] = + offsetof(model_net_wrap_msg, msg.m_torus); + msg_offsets[DRAGONFLY] = + offsetof(model_net_wrap_msg, msg.m_dfly); + msg_offsets[LOGGP] = + offsetof(model_net_wrap_msg, msg.m_loggp); + + // perform the configuration(s) + // This part is tricky, as we basically have to look up all annotations that + // have LP names of the form modelnet_*. For each of those, we need to read + // the base parameters + // - the init is a little easier as we can use the LP-id to look up the + // annotation + + // first grab all of the annotations and store locally + for (int c = 0; c < lpconf.lpannos_count; c++){ + const config_anno_map_t *amap = &lpconf.lpannos[c]; + if (strncmp("modelnet_", amap->lp_name, 9) == 0){ + for (int n = 0; n < amap->num_annos; n++){ + int a; + for (a = 0; a < num_params; a++){ + if (annos[a] != NULL && + strcmp(amap->annotations[n], annos[a]) == 0){ + break; + } + } + if (a == num_params){ + // found a new annotation + annos[num_params++] = amap->annotations[n]; + } + } + if (amap->has_unanno_lp){ + int a; + for (a = 0; a < num_params; a++){ + if (annos[a] == NULL) + break; + } + if (a == num_params){ + // found a new (empty) annotation + annos[num_params++] = NULL; + } + } + } } - // initialize the msg-specific offsets - for (int i = 0; i < id_count; i++){ - switch(ids[i]){ - case SIMPLENET: - msg_offsets[SIMPLENET] = - offsetof(model_net_wrap_msg, msg.m_snet); - break; - case SIMPLEWAN: - msg_offsets[SIMPLEWAN] = - offsetof(model_net_wrap_msg, msg.m_swan); - break; - case TORUS: - msg_offsets[TORUS] = - offsetof(model_net_wrap_msg, msg.m_torus); - break; - case DRAGONFLY: - msg_offsets[DRAGONFLY] = - offsetof(model_net_wrap_msg, msg.m_dfly); - break; - case LOGGP: - msg_offsets[LOGGP] = - offsetof(model_net_wrap_msg, msg.m_loggp); - break; - default: - assert(0); - } + // now that we have all of the annos for all of the networks, loop through + // and read the configs + for (int i = 0; i < num_params; i++){ + base_read_config(annos[i], &all_params[i]); } } @@ -138,14 +240,20 @@ void model_net_base_lp_init( model_net_base_state * ns, tw_lp * lp){ // obtain the underlying lp type through codes-mapping - char lp_type_name[MAX_NAME_LENGTH]; + char lp_type_name[MAX_NAME_LENGTH], anno[MAX_NAME_LENGTH]; int dummy; - // TODO: ignoring annotation to get things up and running, but we'll want - // to actually have a sane design here at some point - // we only need the lp type name for this lp codes_mapping_get_lp_info(lp->gid, NULL, &dummy, - lp_type_name, &dummy, NULL, &dummy, &dummy); + lp_type_name, &dummy, anno, &dummy, &dummy); + + // get annotation-specific parameters + for (int i = 0; i < num_params; i++){ + if ((anno[0]=='\0' && annos[i] == NULL) || + strcmp(anno, annos[i]) == 0){ + ns->params = &all_params[i]; + break; + } + } // find the corresponding method name / index for (int i = 0; i < MAX_NETS; i++){ @@ -157,7 +265,8 @@ void model_net_base_lp_init( // TODO: parameterize scheduler type ns->sched = malloc(sizeof(model_net_sched)); - model_net_sched_init(mn_sched_type, method_array[ns->net_id], ns->sched); + model_net_sched_init(ns->params->stype, method_array[ns->net_id], + ns->sched); ns->sub_type = model_net_get_lp_type(ns->net_id); // NOTE: some models actually expect LP state to be 0 initialized... @@ -236,6 +345,8 @@ void handle_new_msg( tw_lp * lp){ // simply pass down to the scheduler model_net_request *r = &m->msg.m_base.u.req; + // don't forget to set packet size, now that we're responsible for it! + r->packet_size = ns->params->packet_size; void * m_data = m+1; void *remote = NULL, *local = NULL; if (r->remote_event_size > 0){ diff --git a/src/models/networks/model-net/model-net-sched.c b/src/models/networks/model-net/model-net-sched.c index 185c496619d2e35eab9c6ddaf3ca4a7de8b8f86f..be143059081d19427bd1708ee4757563f36cc6dd 100644 --- a/src/models/networks/model-net/model-net-sched.c +++ b/src/models/networks/model-net/model-net-sched.c @@ -14,8 +14,6 @@ #include "codes/model-net-lp.h" #include "codes/quicklist.h" -enum sched_type mn_sched_type = -1; - #define X(a,b,c) b, char * sched_names [] = { SCHEDULER_TYPES diff --git a/src/models/networks/model-net/model-net.c b/src/models/networks/model-net/model-net.c index 913989ec1b43861490178f62e64fbeb0a13ac8f4..0e0f2f731d6bfb5a4c516da6a2e806726d6783ba 100644 --- a/src/models/networks/model-net/model-net.c +++ b/src/models/networks/model-net/model-net.c @@ -12,6 +12,7 @@ #include "codes/model-net-lp.h" #include "codes/model-net-sched.h" #include "codes/codes.h" +#include #define STR_SIZE 16 #define PROC_TIME 10.0 @@ -41,26 +42,83 @@ struct model_net_method* method_array[] = { }; #undef X +// counter and offset for the MN_START_SEQ / MN_END_SEQ macros int in_sequence = 0; tw_stime mn_msg_offset = 0.0; -int model_net_setup(char* name, - uint64_t packet_size, - const void* net_params) -{ - int i; - /* find struct for underlying method (according to configuration file) */ - for(i=0; method_array[i] != NULL; i++) - { - if(strcmp(model_net_method_names[i], name) == 0) - { - method_array[i]->mn_setup(net_params); - method_array[i]->packet_size = packet_size; - return(i); - } - } - fprintf(stderr, "Error: undefined network name %s (Available options simplenet, torus, dragonfly) \n", name); - return -1; // indicating error +// global listing of lp types found by model_net_register +// - needs to be held between the register and configure calls +static int do_config_nets[MAX_NETS]; + +void model_net_register(){ + // first set up which networks need to be registered, then pass off to base + // LP to do its thing + memset(do_config_nets, 0, MAX_NETS * sizeof(*do_config_nets)); + for (int grp = 0; grp < lpconf.lpgroups_count; grp++){ + config_lpgroup_t *lpgroup = &lpconf.lpgroups[grp]; + for (int lpt = 0; lpt < lpgroup->lptypes_count; lpt++){ + char *nm = lpgroup->lptypes[lpt].name; + for (int n = 0; n < MAX_NETS; n++){ + if (!do_config_nets[n] && + strcmp(model_net_lp_config_names[n], nm) == 0){ + do_config_nets[n] = 1; + break; + } + } + } + } + model_net_base_register(do_config_nets); +} + +int* model_net_configure(int *id_count){ + // first call the base LP configure, which sets up the general parameters + model_net_base_configure(); + + // do network-specific configures + *id_count = 0; + for (int i = 0; i < MAX_NETS; i++) { + if (do_config_nets[i]){ + method_array[i]->mn_configure(); + (*id_count)++; + } + } + + // allocate the output + int *ids = malloc(*id_count * sizeof(int)); + // read the ordering provided by modelnet_order + char **values; + size_t length; + int ret = configuration_get_multivalue(&config, "PARAMS", "modelnet_order", + NULL, &values, &length); + if (ret != 1){ + tw_error(TW_LOC, "unable to read PARAMS:modelnet_order variable\n"); + } + if (length != (size_t) *id_count){ + tw_error(TW_LOC, "number of networks in PARAMS:modelnet_order " + "do not match number in LPGROUPS\n"); + } + // set the index + for (int i = 0; i < *id_count; i++){ + ids[i] = -1; + for (int n = 0; n < MAX_NETS; n++){ + if (strcmp(values[i], model_net_method_names[n]) == 0){ + if (!do_config_nets[n]){ + tw_error(TW_LOC, "network in PARAMS:modelnet_order not " + "present in LPGROUPS: %s\n", values[i]); + } + ids[i] = n; + break; + } + } + if (ids[i] == -1){ + tw_error(TW_LOC, "unknown network in PARAMS:modelnet_order: %s\n", + values[i]); + } + free(values[i]); + } + free(values); + + return ids; } int model_net_get_id(char *name){ @@ -193,7 +251,6 @@ static void model_net_event_impl_base( // set the request struct model_net_request *r = &m->msg.m_base.u.req; r->net_id = net_id; - r->packet_size = model_net_get_packet_size(net_id); r->final_dest_lp = final_dest_lp; r->src_lp = sender->gid; r->msg_size = message_size; @@ -251,311 +308,6 @@ void model_net_pull_event( 1, offset, self_event_size, self_event, 0, NULL, sender); } -int* model_net_set_params(int *id_count) -{ - char sched[MAX_NAME_LENGTH]; - long int packet_size_l = 0; - uint64_t packet_size; - int ret; - config_lpgroups_t lpconf; - - // TODO: currently just hard-coding the name determination, should probably - // refactor to do this elsewhere (in codes-base?) - int do_config_nets[MAX_NETS]; - memset(do_config_nets, 0, MAX_NETS*sizeof(*do_config_nets)); - // TODO: inefficient (codes_mapping opens up the same section), but at least - // this avoids the need to call codes_mapping_setup first - configuration_get_lpgroups(&config, "LPGROUPS", &lpconf); - for (int grp = 0; grp < lpconf.lpgroups_count; grp++){ - config_lpgroup_t *lpgroup = &lpconf.lpgroups[grp]; - for (int lpt = 0; lpt < lpgroup->lptypes_count; lpt++){ - char *nm = lpgroup->lptypes[lpt].name; - for (int n = 0; n < MAX_NETS; n++){ - if (strcmp(model_net_lp_config_names[n], nm) == 0){ - do_config_nets[n] = 1; - break; - } - } - } - } - - ret = configuration_get_value(&config, "PARAMS", "modelnet_scheduler", NULL, - sched, MAX_NAME_LENGTH); - - configuration_get_value_longint(&config, "PARAMS", "packet_size", NULL, - &packet_size_l); - packet_size = packet_size_l; - - if (ret > 0){ - int i; - for (i = 0; i < MAX_SCHEDS; i++){ - if (strcmp(sched_names[i], sched) == 0){ - mn_sched_type = i; - break; - } - } - if (i == MAX_SCHEDS){ - fprintf(stderr, - "Unknown value for PARAMS:modelnet-scheduler : %s\n", - sched); - abort(); - } - } - else{ - // default: FCFS - mn_sched_type = MN_SCHED_FCFS; - } - - if (mn_sched_type == MN_SCHED_FCFS_FULL){ - // override packet size to something huge (leave a bit in the unlikely - // case that an op using packet size causes overflow) - packet_size = 1ull << 62; - } - else if (!packet_size && mn_sched_type != MN_SCHED_FCFS_FULL) - { - packet_size = 512; - fprintf(stderr, "\n Warning, no packet size specified, setting packet size to %llu\n", packet_size); - } - - if(do_config_nets[SIMPLENET]) - { - double net_startup_ns, net_bw_mbps; - simplenet_param net_params; - - configuration_get_value_double(&config, "PARAMS", "net_startup_ns", NULL, - &net_startup_ns); - configuration_get_value_double(&config, "PARAMS", "net_bw_mbps", NULL, - &net_bw_mbps); - net_params.net_startup_ns = net_startup_ns; - net_params.net_bw_mbps = net_bw_mbps; - model_net_setup(model_net_method_names[SIMPLENET], packet_size, (const void*)&net_params); /* Sets the network as simplenet and packet size 512 */ - } - if (do_config_nets[SIMPLEWAN]){ - simplewan_param net_params; - configuration_get_value_relpath(&config, "PARAMS", "net_startup_ns_file", NULL, net_params.startup_filename, MAX_NAME_LENGTH); - configuration_get_value_relpath(&config, "PARAMS", "net_bw_mbps_file", NULL, net_params.bw_filename, MAX_NAME_LENGTH); - model_net_setup(model_net_method_names[SIMPLEWAN], packet_size, (const void*)&net_params); - } - if(do_config_nets[LOGGP]) - { - char net_config_file[256]; - loggp_param net_params; - - configuration_get_value_relpath(&config, "PARAMS", "net_config_file", NULL, net_config_file, 256); - net_params.net_config_file = net_config_file; - model_net_setup(model_net_method_names[LOGGP], packet_size, (const void*)&net_params); /* Sets the network as loggp and packet size 512 */ - } - - if(do_config_nets[DRAGONFLY]) - { - dragonfly_param net_params; - int num_routers=0, num_vcs=0, local_vc_size=0, global_vc_size=0, cn_vc_size=0, chunk_size=0; - double local_bandwidth=0.0, cn_bandwidth=0.0, global_bandwidth=0.0; - - configuration_get_value_int(&config, "PARAMS", "num_routers", NULL, &num_routers); - if(!num_routers) - { - num_routers = 4; - printf("\n Number of dimensions not specified, setting to %d ", num_routers); - } - net_params.num_routers = num_routers; - - configuration_get_value_int(&config, "PARAMS", "num_vcs", NULL, &num_vcs); - if(!num_vcs) - { - num_vcs = 1; - printf("\n Number of virtual channels not specified, setting to %d ", num_vcs); - } - net_params.num_vcs = num_vcs; - - configuration_get_value_int(&config, "PARAMS", "local_vc_size", NULL, &local_vc_size); - if(!local_vc_size) - { - local_vc_size = 1024; - printf("\n Buffer size of local channels not specified, setting to %d ", local_vc_size); - } - net_params.local_vc_size = local_vc_size; - - configuration_get_value_int(&config, "PARAMS", "global_vc_size", NULL, &global_vc_size); - if(!global_vc_size) - { - global_vc_size = 2048; - printf("\n Buffer size of global channels not specified, setting to %d ", global_vc_size); - } - net_params.global_vc_size = global_vc_size; - - configuration_get_value_int(&config, "PARAMS", "cn_vc_size", NULL, &cn_vc_size); - if(!cn_vc_size) - { - cn_vc_size = 1024; - printf("\n Buffer size of compute node channels not specified, setting to %d ", cn_vc_size); - } - net_params.cn_vc_size = cn_vc_size; - - configuration_get_value_int(&config, "PARAMS", "chunk_size", NULL, &chunk_size); - if(!chunk_size) - { - chunk_size = 64; - printf("\n Chunk size for packets is specified, setting to %d ", chunk_size); - } - net_params.chunk_size = chunk_size; - - configuration_get_value_double(&config, "PARAMS", "local_bandwidth", NULL, &local_bandwidth); - if(!local_bandwidth) - { - local_bandwidth = 5.25; - printf("\n Bandwidth of local channels not specified, setting to %lf ", local_bandwidth); - } - net_params.local_bandwidth = local_bandwidth; - - configuration_get_value_double(&config, "PARAMS", "global_bandwidth", NULL, &global_bandwidth); - if(!global_bandwidth) - { - global_bandwidth = 4.7; - printf("\n Bandwidth of global channels not specified, setting to %lf ", global_bandwidth); - } - net_params.global_bandwidth = global_bandwidth; - - configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", NULL, &cn_bandwidth); - if(!cn_bandwidth) - { - cn_bandwidth = 5.25; - printf("\n Bandwidth of compute node channels not specified, setting to %lf ", cn_bandwidth); - } - net_params.cn_bandwidth = cn_bandwidth; - - - char routing[MAX_NAME_LENGTH]; - configuration_get_value(&config, "PARAMS", "routing", NULL, routing, MAX_NAME_LENGTH); - if(strcmp(routing, "minimal") == 0) - net_params.routing = 0; - else if(strcmp(routing, "nonminimal")==0 || strcmp(routing,"non-minimal")==0) - net_params.routing = 1; - else if (strcmp(routing, "adaptive") == 0) - net_params.routing = 2; - else - { - printf("\n No routing protocol specified, setting to minimal routing"); - net_params.routing = 0; - } - model_net_setup(model_net_method_names[DRAGONFLY], packet_size, (const void*)&net_params); - } - if(do_config_nets[TORUS]) - { - torus_param net_params; - char dim_length[MAX_NAME_LENGTH]; - int n_dims=0, buffer_size=0, num_vc=0, i=0, chunk_size = 0; - double link_bandwidth=0; - - configuration_get_value_int(&config, "PARAMS", "n_dims", NULL, &n_dims); - if(!n_dims) - { - n_dims = 4; /* a 4-D torus */ - printf("\n Number of dimensions not specified, setting to %d ", n_dims); - } - - configuration_get_value_double(&config, "PARAMS", "link_bandwidth", NULL, &link_bandwidth); - if(!link_bandwidth) - { - link_bandwidth = 2.0; /*default bg/q configuration */ - printf("\n Link bandwidth not specified, setting to %lf ", link_bandwidth); - } - - configuration_get_value_int(&config, "PARAMS", "buffer_size", NULL, &buffer_size); - if(!buffer_size) - { - buffer_size = 2048; - printf("\n Buffer size not specified, setting to %d ",buffer_size); - } - - configuration_get_value_int(&config, "PARAMS", "chunk_size", NULL, &chunk_size); - if(!chunk_size) - { - chunk_size = 32; - printf("\n Chunk size not specified, setting to %d ", chunk_size); - } - configuration_get_value_int(&config, "PARAMS", "num_vc", NULL, &num_vc); - if(!num_vc) - { - num_vc = 1; /*by default, we have one for taking packets, another for taking credit*/ - printf("\n num_vc not specified, setting to %d ", num_vc); - } - - configuration_get_value(&config, "PARAMS", "dim_length", NULL, dim_length, MAX_NAME_LENGTH); - char* token; - net_params.n_dims=n_dims; - net_params.num_vc=num_vc; - net_params.buffer_size=buffer_size; - net_params.chunk_size = chunk_size; - net_params.link_bandwidth=link_bandwidth; - net_params.dim_length=malloc(n_dims*sizeof(int)); - token = strtok(dim_length, ","); - while(token != NULL) - { - sscanf(token, "%d", &net_params.dim_length[i]); - if(!net_params.dim_length[i]) - { - printf("\n Invalid torus dimension specified %d, exitting... ", net_params.dim_length[i]); - MPI_Finalize(); - exit(-1); - } - i++; - token = strtok(NULL,","); - } - model_net_setup(model_net_method_names[TORUS], packet_size, (const void*)&net_params); - } - - // now that the LP-specific nets are set up... - // - get the number of nets used - *id_count = 0; - for (int i = 0; i < MAX_NETS; i++){ - if (do_config_nets[i]){ - (*id_count)++; - } - } - // - allocate the output - int *ids = malloc(*id_count * sizeof(int)); - // - read the ordering provided by modelnet_order - char **values; - size_t length; - ret = configuration_get_multivalue(&config, "PARAMS", "modelnet_order", NULL, &values, &length); - if (ret != 1){ - fprintf(stderr, "unable to read PARAMS:modelnet_order variable\n"); - abort(); - } - if (length != (size_t) *id_count){ - fprintf(stderr, "number of networks in PARAMS:modelnet_order " - "do not match number in LPGROUPS\n"); - abort(); - } - // - set the index - for (int i = 0; i < *id_count; i++){ - ids[i] = -1; - for (int n = 0; n < MAX_NETS; n++){ - if (strcmp(values[i], model_net_method_names[n]) == 0){ - if (!do_config_nets[n]){ - fprintf(stderr, "network in PARAMS:modelnet_order not " - "present in LPGROUPS: %s\n", values[i]); - abort(); - } - ids[i] = n; - break; - } - } - if (ids[i] == -1){ - fprintf(stderr, "unknown network in PARAMS:modelnet_order: %s\n", - values[i]); - abort(); - } - free(values[i]); - } - free(values); - // - pass along to model_net_base_init - model_net_base_init(*id_count, ids); - // - done - return ids; -} - void model_net_event_rc( int net_id, tw_lp *sender, diff --git a/src/models/networks/model-net/simplenet-upd.c b/src/models/networks/model-net/simplenet-upd.c index 369c9dc86f1c1bf0bfc38a892424eb75bf5bc294..9c7731bab30c1e8d2c7d31964808ba0e32eb7083 100644 --- a/src/models/networks/model-net/simplenet-upd.c +++ b/src/models/networks/model-net/simplenet-upd.c @@ -23,6 +23,14 @@ #define LP_CONFIG_NM (model_net_lp_config_names[SIMPLENET]) #define LP_METHOD_NM (model_net_method_names[SIMPLENET]) +/* structs for initializing a network/ specifying network parameters */ +struct simplenet_param +{ + double net_startup_ns; /*simplenet startup cost*/ + double net_bw_mbps; /*Link bandwidth per byte*/ +}; +typedef struct simplenet_param simplenet_param; + /*Define simplenet data types and structs*/ typedef struct sn_state sn_state; @@ -31,26 +39,22 @@ struct sn_state /* next idle times for network card, both inbound and outbound */ tw_stime net_send_next_idle; tw_stime net_recv_next_idle; + const char * anno; + simplenet_param params; struct mn_stats sn_stats_array[CATEGORY_MAX]; }; -/* net startup cost, ns */ -static double global_net_startup_ns = 0; -/* net bw, MB/s */ -static double global_net_bw_mbs = 0; +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static uint64_t num_params = 0; +static simplenet_param * all_params = NULL; +static const config_anno_map_t * anno_map = NULL; static int sn_magic = 0; /* returns a pointer to the lptype struct to use for simplenet LPs */ static const tw_lptype* sn_get_lp_type(void); -/* set model parameters: - * - * - net_startup_ns: network startup cost in ns. - * - net_bs_mbs: network bandwidth in MiB/s (megabytes per second). - */ -static void sn_set_params(double net_startup_ns, double net_bw_mbs); - /* retrieve the size of the portion of the event struct that is consumed by * the simplenet module. The caller should add this value to the size of * its own event structure to get the maximum total size of a message. @@ -84,7 +88,7 @@ static void simple_net_collective_rc(); */ /* Modelnet interface events */ /* sets up the simplenet parameters through modelnet interface */ -static void sn_setup(const void* net_params); +static void sn_configure(); /* Issues a simplenet packet event call */ static tw_stime simplenet_packet_event( @@ -112,7 +116,7 @@ static tw_lpid sn_find_local_device(tw_lp *sender); /* data structure for model-net statistics */ struct model_net_method simplenet_method = { - .mn_setup = sn_setup, + .mn_configure = sn_configure, .model_net_method_packet_event = simplenet_packet_event, .model_net_method_packet_event_rc = simplenet_packet_event_rc, .mn_get_lp_type = sn_get_lp_type, @@ -198,17 +202,6 @@ static void simple_net_collective_rc() return; } -/* lets caller specify model parameters to use */ -static void sn_set_params(double net_startup_ns, double net_bw_mbs) -{ - assert(net_startup_ns > 0); - assert(net_bw_mbs > 0); - global_net_startup_ns = net_startup_ns; - global_net_bw_mbs = net_bw_mbs; - - return; -} - /* report network statistics */ static void sn_report_stats() { @@ -226,6 +219,14 @@ static void sn_init( ns->net_send_next_idle = tw_now(lp); ns->net_recv_next_idle = tw_now(lp); + ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid); + if (ns->anno == NULL) + ns->params = all_params[num_params-1]; + else{ + int id = configuration_get_annotation_index(ns->anno, anno_map); + ns->params = all_params[id]; + } + bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2); sn_magic = h1+h2; /* printf("\n sn_magic %d ", sn_magic); */ @@ -349,7 +350,8 @@ static void handle_msg_ready_event( stat->recv_count++; stat->recv_bytes += m->net_msg_size_bytes; m->recv_time_saved = stat->recv_time; - stat->recv_time += rate_to_ns(m->net_msg_size_bytes, global_net_bw_mbs); + stat->recv_time += rate_to_ns(m->net_msg_size_bytes, + ns->params.net_bw_mbps); /* are we available to recv the msg? */ /* were we available when the transmission was started? */ @@ -357,7 +359,8 @@ static void handle_msg_ready_event( recv_queue_time += ns->net_recv_next_idle - tw_now(lp); /* calculate transfer time based on msg size and bandwidth */ - recv_queue_time += rate_to_ns(m->net_msg_size_bytes, global_net_bw_mbs); + recv_queue_time += rate_to_ns(m->net_msg_size_bytes, + ns->params.net_bw_mbps); /* bump up input queue idle time accordingly */ m->net_recv_next_idle_saved = ns->net_recv_next_idle; @@ -439,12 +442,13 @@ static void handle_msg_start_event( stat->send_count++; stat->send_bytes += m->net_msg_size_bytes; m->send_time_saved = stat->send_time; - stat->send_time += (global_net_startup_ns + rate_to_ns(m->net_msg_size_bytes, global_net_bw_mbs)); + stat->send_time += (ns->params.net_startup_ns + rate_to_ns(m->net_msg_size_bytes, + ns->params.net_bw_mbps)); if(stat->max_event_size < total_event_size) stat->max_event_size = total_event_size; /* calculate send time stamp */ - send_queue_time = global_net_startup_ns; /* net msg startup cost */ + send_queue_time = ns->params.net_startup_ns; /* net msg startup cost */ /* bump up time if the NIC send queue isn't idle right now */ if(ns->net_send_next_idle > tw_now(lp)) send_queue_time += ns->net_send_next_idle - tw_now(lp); @@ -454,14 +458,14 @@ static void handle_msg_start_event( */ m->net_send_next_idle_saved = ns->net_send_next_idle; ns->net_send_next_idle = send_queue_time + tw_now(lp) + - rate_to_ns(m->net_msg_size_bytes, global_net_bw_mbs); + rate_to_ns(m->net_msg_size_bytes, ns->params.net_bw_mbps); /* create new event to send msg to receiving NIC */ // TODO: don't ignore annotations codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &dummy, NULL, &dummy, NULL, &mapping_rep_id, &mapping_offset); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1, + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0, mapping_rep_id, mapping_offset, &dest_id); // printf("\n msg start sending to %d ", dest_id); @@ -569,10 +573,43 @@ static tw_stime simplenet_packet_event( return xfer_to_nic_time; } -static void sn_setup(const void* net_params) +static void sn_configure() { - simplenet_param* sn_param = (simplenet_param*)net_params; - sn_set_params(sn_param->net_startup_ns, sn_param->net_bw_mbps); + anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM); + assert(anno_map); + num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0); + all_params = malloc(num_params * sizeof(*all_params)); + for (uint64_t i = 0; i < anno_map->num_annos; i++){ + const char * anno = anno_map->annotations[i]; + int rc; + rc = configuration_get_value_double(&config, "PARAMS", + "net_startup_ns", anno, &all_params[i].net_startup_ns); + if (rc != 0){ + tw_error(TW_LOC, + "simplenet: unable to read PARAMS:net_startup_ns@%s", + anno); + } + rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps", + anno, &all_params[i].net_bw_mbps); + if (rc != 0){ + tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps@%s", + anno); + } + } + if (anno_map->has_unanno_lp > 0){ + int rc; + rc = configuration_get_value_double(&config, "PARAMS", + "net_startup_ns", NULL, + &all_params[num_params-1].net_startup_ns); + if (rc != 0){ + tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_startup_ns"); + } + rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps", + NULL, &all_params[num_params-1].net_bw_mbps); + if (rc != 0){ + tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps"); + } + } } static void simplenet_packet_event_rc(tw_lp *sender) diff --git a/src/models/networks/model-net/simplewan.c b/src/models/networks/model-net/simplewan.c index 217cd555edf82d138f9c41168b8e0083a3f78791..e995cc00e799ea451250e1243cb22cf2065953ff 100644 --- a/src/models/networks/model-net/simplewan.c +++ b/src/models/networks/model-net/simplewan.c @@ -25,6 +25,16 @@ #define LP_CONFIG_NM (model_net_lp_config_names[SIMPLEWAN]) #define LP_METHOD_NM (model_net_method_names[SIMPLEWAN]) +// parameters for simplewan configuration +struct simplewan_param +{ + double * net_startup_ns_table; + double * net_bw_mbps_table; + int mat_len; + int num_lps; +}; +typedef struct simplewan_param simplewan_param; + /*Define simplewan data types and structs*/ typedef struct sw_state sw_state; @@ -46,6 +56,9 @@ struct sw_state tw_stime *send_next_idle; tw_stime *recv_next_idle; + const char * anno; + const simplewan_param * params; + int id; /* logical id for matrix lookups */ /* Each simplewan "NIC" actually has N connections, so we need to track @@ -57,14 +70,11 @@ struct sw_state struct mn_stats sw_stats_array[CATEGORY_MAX]; }; -/* net startup cost, ns */ -static double *global_net_startup_ns = NULL; -/* net bw, MB/s */ -static double *global_net_bw_mbs = NULL; -static int mat_len = -1; -/* number of simplewan lps, used for addressing the network parameters - * set by the first LP to init per process */ -static int num_lps = -1; +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static uint64_t num_params = 0; +static simplewan_param * all_params = NULL; +static const config_anno_map_t * anno_map = NULL; static int sw_magic = 0; @@ -76,7 +86,12 @@ static const tw_lptype* sw_get_lp_type(void); * - bw_fname - path containing triangular matrix of bandwidths in MB/s. * note that this merely stores the files, they will be parsed later */ -static void sw_set_params(char * startup_fname, char * bw_fname); +static void sw_set_params( + const char * startup_fname, + const char * bw_fname, + simplewan_param * params); + +static void sw_configure(); /* retrieve the size of the portion of the event struct that is consumed by * the simplewan module. The caller should add this value to the size of @@ -89,8 +104,11 @@ static int sw_get_magic(); /* given two simplewan logical ids, do matrix lookups to get the point-to-point * latency/bandwidth */ -static double sw_get_bw(int from_id, int to_id); -static double sw_get_startup(int from_id, int to_id); +static double sw_get_table_ent( + int from_id, + int to_id, + int num_lps, + double * table); /* category lookup */ static category_idles* sw_get_category_idles( @@ -102,22 +120,6 @@ static void simple_wan_collective(); /* collective network calls-- rc */ static void simple_wan_collective_rc(); -/* allocate a new event that will pass through simplewan to arriave at its - * destination: - * - * - category: category name to associate with this communication - * - final_dest_gid: the LP that the message should be delivered to. - * - event_size_bytes: size of event msg that will be delivered to - * final_dest_gid. - * - local_event_size_byte: size of event message that will delivered to - * local LP upon local send comletion (set to 0 if not used) - * - net_msg_size_bytes: size of simulated network message in bytes. - * - sender: LP calling this function. - */ -/* Modelnet interface events */ -/* sets up the simplewan parameters through modelnet interface */ -static void sw_setup(const void* net_params); - /* Issues a simplewan packet event call */ static tw_stime simplewan_packet_event( char* category, @@ -144,13 +146,13 @@ static tw_lpid sw_find_local_device(tw_lp *sender); /* data structure for model-net statistics */ struct model_net_method simplewan_method = { - .mn_setup = sw_setup, + .mn_configure = sw_configure, .model_net_method_packet_event = simplewan_packet_event, .model_net_method_packet_event_rc = simplewan_packet_event_rc, .mn_get_lp_type = sw_get_lp_type, .mn_get_msg_sz = sw_get_msg_sz, .mn_report_stats = sw_report_stats, - .model_net_method_find_local_device = sw_find_local_device, + .model_net_method_find_local_device = sw_find_local_device, .mn_collective_call = simple_wan_collective, .mn_collective_call_rc = simple_wan_collective_rc }; @@ -292,7 +294,10 @@ static void fill_tri_mat(int N, double *mat, double *tri){ } /* lets caller specify model parameters to use */ -static void sw_set_params(char * startup_fname, char * bw_fname){ +static void sw_set_params( + const char * startup_fname, + const char * bw_fname, + simplewan_param * params){ long int fsize_s, fsize_b; /* TODO: make this a run-time option */ int is_tri_mat = 0; @@ -300,6 +305,10 @@ static void sw_set_params(char * startup_fname, char * bw_fname){ /* slurp the files */ FILE *sf = fopen(startup_fname, "r"); FILE *bf = fopen(bw_fname, "r"); + if (!sf) + tw_error(TW_LOC, "simplewan: unable to open %s", startup_fname); + if (!bf) + tw_error(TW_LOC, "simplewan: unable to open %s", bw_fname); fseek(sf, 0, SEEK_END); fsize_s = ftell(sf); fseek(sf, 0, SEEK_SET); @@ -323,18 +332,20 @@ static void sw_set_params(char * startup_fname, char * bw_fname){ /* convert tri mat into a regular mat */ assert(nvals_first_s == nvals_first_b); - mat_len = nvals_first_s + ((is_tri_mat) ? 1 : 0); + params->mat_len = nvals_first_s + ((is_tri_mat) ? 1 : 0); if (is_tri_mat){ - global_net_startup_ns = malloc(mat_len*mat_len*sizeof(double)); - global_net_bw_mbs = malloc(mat_len*mat_len*sizeof(double)); - fill_tri_mat(mat_len, global_net_startup_ns, startup_tmp); - fill_tri_mat(mat_len, global_net_bw_mbs, bw_tmp); + params->net_startup_ns_table = + malloc(params->mat_len*params->mat_len*sizeof(double)); + params->net_bw_mbps_table = + malloc(params->mat_len*params->mat_len*sizeof(double)); + fill_tri_mat(params->mat_len, params->net_startup_ns_table, startup_tmp); + fill_tri_mat(params->mat_len, params->net_bw_mbps_table, bw_tmp); free(startup_tmp); free(bw_tmp); } else{ - global_net_startup_ns = startup_tmp; - global_net_bw_mbs = bw_tmp; + params->net_startup_ns_table = startup_tmp; + params->net_bw_mbps_table = bw_tmp; } /* done */ @@ -357,30 +368,25 @@ static void sw_init( sw_magic = h1+h2; /* printf("\n sw_magic %d ", sw_magic); */ - /* inititalize global logical ID - * TODO: be annotation aware */ - ns->id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); - - /* get the total number of simplewans */ - if (num_lps == -1){ - num_lps = codes_mapping_get_lp_count(NULL, 0, LP_CONFIG_NM, NULL, 1); - assert(num_lps > 0); - if (mat_len == -1){ - tw_error(TW_LOC, "Simplewan config matrix not initialized " - "at lp init time\n"); - } - else if (mat_len != num_lps){ - tw_error(TW_LOC, "Simplewan config matrix doesn't match the " - "number of simplewan LPs\n"); - } + ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid); + if (ns->anno == NULL) + ns->params = &all_params[num_params-1]; + else{ + int id = configuration_get_annotation_index(ns->anno, anno_map); + ns->params = &all_params[id]; } + /* inititalize global logical ID w.r.t. annotation */ + ns->id = codes_mapping_get_lp_relative_id(lp->gid, 0, 1); + /* all devices are idle to begin with */ - ns->send_next_idle = malloc(num_lps * sizeof(ns->send_next_idle)); - ns->recv_next_idle = malloc(num_lps * sizeof(ns->recv_next_idle)); + ns->send_next_idle = malloc(ns->params->num_lps * + sizeof(ns->send_next_idle)); + ns->recv_next_idle = malloc(ns->params->num_lps * + sizeof(ns->recv_next_idle)); tw_stime st = tw_now(lp); int i; - for (i = 0; i < num_lps; i++){ + for (i = 0; i < ns->params->num_lps; i++){ ns->send_next_idle[i] = st; ns->recv_next_idle[i] = st; } @@ -524,8 +530,10 @@ static void handle_msg_ready_event( struct mn_stats* stat; /* get source->me network stats */ - double bw = sw_get_bw(m->src_mn_rel_id, ns->id); - double startup = sw_get_startup(m->src_mn_rel_id, ns->id); + double bw = sw_get_table_ent(m->src_mn_rel_id, ns->id, + ns->params->num_lps, ns->params->net_bw_mbps_table); + double startup = sw_get_table_ent(m->src_mn_rel_id, ns->id, + ns->params->num_lps, ns->params->net_startup_ns_table); if (bw <= 0.0 || startup < 0.0){ fprintf(stderr, "Invalid link from Rel. id %d to LP %lu (rel. id %d)\n", @@ -662,17 +670,18 @@ static void handle_msg_start_event( total_event_size = model_net_get_msg_sz(SIMPLEWAN) + m->event_size_bytes + m->local_event_size_bytes; - // TODO: don't ignore annotations codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &dummy, NULL, &dummy, NULL, &mapping_rep_id, &mapping_offset); - codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1, + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0, mapping_rep_id, mapping_offset, &dest_id); dest_rel_id = codes_mapping_get_lp_relative_id(dest_id, 0, 0); m->dest_mn_rel_id = dest_rel_id; /* grab the link params */ - bw = sw_get_bw(ns->id, dest_rel_id); - startup = sw_get_startup(ns->id, dest_rel_id); + bw = sw_get_table_ent(ns->id, dest_rel_id, + ns->params->num_lps, ns->params->net_bw_mbps_table); + startup = sw_get_table_ent(ns->id, dest_rel_id, + ns->params->num_lps, ns->params->net_startup_ns_table); if (bw <= 0.0 || startup < 0.0){ fprintf(stderr, "Invalid link from LP %lu (rel. id %d) to LP %lu (rel. id %d)\n", @@ -846,11 +855,53 @@ static tw_stime simplewan_packet_event( return xfer_to_nic_time; } -static void sw_setup(const void* net_params) -{ - simplewan_param* sw_param = (simplewan_param*)net_params; - sw_set_params(sw_param->startup_filename, sw_param->bw_filename); - //printf("\n Bandwidth setup %lf %lf ", sw_param->net_startup_ns, sw_param->net_bw_mbps); +static void sw_read_config(const char * anno, simplewan_param *p){ + char startup_file[MAX_NAME_LENGTH]; + char bw_file[MAX_NAME_LENGTH]; + int rc; + rc = configuration_get_value_relpath(&config, "PARAMS", + "net_startup_ns_file", anno, startup_file, MAX_NAME_LENGTH); + if (rc == 0){ + if (anno == NULL) + tw_error(TW_LOC, + "simplewan: unable to read PARAMS:net_startup_ns_file"); + else + tw_error(TW_LOC, + "simplewan: unable to read PARAMS:net_startup_ns_file@%s", + anno); + } + rc = configuration_get_value_relpath(&config, "PARAMS", "net_bw_mbps_file", + anno, bw_file, MAX_NAME_LENGTH); + if (rc == 0){ + if (anno == NULL) + tw_error(TW_LOC, + "simplewan: unable to read PARAMS:net_bw_mbps_file"); + else + tw_error(TW_LOC, + "simplewan: unable to read PARAMS:net_bw_mbps_file@%s", + anno); + } + p->num_lps = codes_mapping_get_lp_count(NULL, 0, + LP_CONFIG_NM, anno, 0); + sw_set_params(startup_file, bw_file, p); + if (p->mat_len != p->num_lps){ + tw_error(TW_LOC, "Simplewan config matrix doesn't match the " + "number of simplewan LPs (%d vs. %d)\n", + p->mat_len, p->num_lps); + } +} + +static void sw_configure(){ + anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM); + assert(anno_map); + num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0); + all_params = malloc(num_params * sizeof(*all_params)); + for (uint64_t i = 0; i < anno_map->num_annos; i++){ + sw_read_config(anno_map->annotations[i], &all_params[i]); + } + if (anno_map->has_unanno_lp > 0){ + sw_read_config(NULL, &all_params[anno_map->num_annos]); + } } static void simplewan_packet_event_rc(tw_lp *sender) @@ -872,11 +923,13 @@ static tw_lpid sw_find_local_device(tw_lp *sender) return(dest_id); } -static double sw_get_bw(int from_id, int to_id){ - return global_net_bw_mbs[from_id * num_lps + to_id]; -} -static double sw_get_startup(int from_id, int to_id){ - return global_net_startup_ns[from_id * num_lps + to_id]; +static double sw_get_table_ent( + int from_id, + int to_id, + int num_lps, + double * table){ + // TODO: if a tri-matrix, then change the addressing + return table[from_id * num_lps + to_id]; } /* category lookup (more or less copied from model_net_find_stats) */ diff --git a/src/models/networks/model-net/torus.c b/src/models/networks/model-net/torus.c index bba4f2f5172209b76ea2d3f01fd24328e7c4776c..e916c1f8f500fd88cfce8f22c1609837719a36c2 100644 --- a/src/models/networks/model-net/torus.c +++ b/src/models/networks/model-net/torus.c @@ -6,6 +6,7 @@ #include #include +#include #include "codes/lp-io.h" #include "codes/codes_mapping.h" @@ -18,7 +19,7 @@ #define CHUNK_SIZE 32 #define DEBUG 1 #define MEAN_INTERVAL 100 -#define TRACE -1 +#define TRACE -1 /* collective specific parameters */ #define TREE_DEGREE 4 @@ -35,22 +36,27 @@ static double maxd(double a, double b) { return a < b ? b : a; } /* Torus network model implementation of codes, implements the modelnet API */ -/* Link bandwidth for each torus link, configurable from the config file */ -static double link_bandwidth; -/* buffer size of each torus link, configurable */ -static int buffer_size; -/* number of virtual channels for each torus link, configurable */ -static int num_vc; -/* number of torus dimensions, configurable */ -static int n_dims; -/* length of each torus dimension, configurable */ -static int * dim_length; -/* factor, used in torus coordinate calculation */ -static int * factor; -/* half length of each dimension, used in torus coordinates calculation */ -static int * half_length; -/* size of each torus chunk, by default it is set to 32 */ -static int chunk_size; +typedef struct torus_param torus_param; +struct torus_param +{ + int n_dims; /*Dimension of the torus network, 5-D, 7-D or any other*/ + int* dim_length; /*Length of each torus dimension*/ + double link_bandwidth;/* bandwidth for each torus link */ + int buffer_size; /* number of buffer slots for each vc in flits*/ + int num_vc; /* number of virtual channels for each torus link */ + float mean_process;/* mean process time for each flit */ + int chunk_size; /* chunk is the smallest unit--default set to 32 */ + + /* "derived" torus parameters */ + + /* factor, used in torus coordinate calculation */ + int * factor; + /* half length of each dimension, used in torus coordinates calculation */ + int * half_length; + + double head_delay; + double credit_delay; +}; /* codes mapping group name, lp type name */ static char grp_name[MAX_NAME_LENGTH]; @@ -62,17 +68,16 @@ static tw_stime total_time = 0; static tw_stime max_latency = 0; static tw_stime max_collective = 0; -/* indicates delays calculated through the bandwidth calculation of the torus link */ -static float head_delay=0.0; -static float credit_delay = 0.0; - /* number of finished packets on each PE */ static long long N_finished_packets = 0; /* total number of hops traversed by a message on each PE */ static long long total_hops = 0; -/* number of chunks/flits in each torus packet, calculated through the size of each flit (32 bytes by default) */ -static uint64_t num_chunks; +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static uint64_t num_params = 0; +static torus_param * all_params = NULL; +static const config_anno_map_t * anno_map = NULL; typedef struct nodes_state nodes_state; @@ -123,29 +128,110 @@ struct nodes_state /* to maintain a count of child nodes that have fanned in at the parent during the collective fan-in phase*/ int num_fan_nodes; + + /* LPs annotation */ + const char * anno; + /* LPs configuration */ + const torus_param * params; }; -/* setup the torus model, initialize global parameters */ -static void torus_setup(const void* net_params) -{ +static void torus_read_config( + const char * anno, + torus_param * params){ + char dim_length_str[MAX_NAME_LENGTH]; int i; - torus_param* t_param = (torus_param*)net_params; - n_dims = t_param->n_dims; - link_bandwidth = t_param->link_bandwidth; - buffer_size = t_param->buffer_size; - num_vc = t_param->num_vc; - chunk_size = t_param->chunk_size; - head_delay = (1 / link_bandwidth) * chunk_size; - credit_delay = (1 / link_bandwidth) * 8; - dim_length = malloc(n_dims * sizeof(int)); - factor = malloc(n_dims * sizeof(int)); - half_length = malloc(n_dims * sizeof(int)); - - for(i = 0; i < n_dims; i++) + + // shorthand + torus_param *p = params; + + configuration_get_value_int(&config, "PARAMS", "n_dims", anno, &p->n_dims); + if(!p->n_dims) { + p->n_dims = 4; /* a 4-D torus */ + fprintf(stderr, + "Warning: Number of dimensions not specified, setting to %d\n", + p->n_dims); + } + + configuration_get_value_double(&config, "PARAMS", "link_bandwidth", anno, + &p->link_bandwidth); + if(!p->link_bandwidth) { + p->link_bandwidth = 2.0; /*default bg/q configuration */ + fprintf(stderr, "Link bandwidth not specified, setting to %lf\n", + p->link_bandwidth); + } + + configuration_get_value_int(&config, "PARAMS", "buffer_size", anno, &p->buffer_size); + if(!p->buffer_size) { + p->buffer_size = 2048; + fprintf(stderr, "Buffer size not specified, setting to %d", + p->buffer_size); + } + + configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size); + if(!p->chunk_size) { + p->chunk_size = 32; + fprintf(stderr, "Warning: Chunk size not specified, setting to %d\n", + p->chunk_size); + } + configuration_get_value_int(&config, "PARAMS", "num_vc", anno, &p->num_vc); + if(!p->num_vc) { + /* by default, we have one for taking packets, + * another for taking credit*/ + p->num_vc = 1; + fprintf(stderr, "Warning: num_vc not specified, setting to %d\n", + p->num_vc); + } + + int rc = configuration_get_value(&config, "PARAMS", "dim_length", anno, + dim_length_str, MAX_NAME_LENGTH); + if (rc == 0){ + tw_error(TW_LOC, "couldn't read PARAMS:dim_length"); + } + char* token; + p->dim_length=malloc(p->n_dims*sizeof(*p->dim_length)); + token = strtok(dim_length_str, ","); + i = 0; + while(token != NULL) { - dim_length[i] = t_param->dim_length[i]; - if(!dim_length[i]) - dim_length[i] = 8; + sscanf(token, "%d", &p->dim_length[i]); + if(p->dim_length[i] <= 0) + { + tw_error(TW_LOC, "Invalid torus dimension specified " + "(%d at pos %d), exiting... ", p->dim_length[i], i); + } + i++; + token = strtok(NULL,","); + } + + // create derived parameters + + // factor is an exclusive prefix product + p->factor = malloc(p->n_dims * sizeof(int)); + p->factor[0] = 1; + for(i = 1; i < p->n_dims; i++) + p->factor[i] = p->factor[i-1] * p->dim_length[i-1]; + + p->half_length = malloc(p->n_dims * sizeof(int)); + for (i = 0; i < p->n_dims; i++) + p->half_length[i] = p->dim_length[i] / 2; + + // some latency numbers + p->head_delay = (1.0 / p->link_bandwidth) * p->chunk_size; + p->credit_delay = (1.0 / p->link_bandwidth) * p->chunk_size; +} + +static void torus_configure(){ + anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM); + assert(anno_map); + num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0); + all_params = malloc(num_params * sizeof(*all_params)); + + for (uint64_t i = 0; i < anno_map->num_annos; i++){ + const char * anno = anno_map->annotations[i]; + torus_read_config(anno, &all_params[i]); + } + if (anno_map->has_unanno_lp > 0){ + torus_read_config(NULL, &all_params[anno_map->num_annos]); } } @@ -154,7 +240,7 @@ void torus_collective_init(nodes_state * s, { // TODO: be annotation-aware somehow codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); - int num_lps = codes_mapping_get_lp_count(grp_name, 1, LP_CONFIG_NM, NULL, 1); + int num_lps = codes_mapping_get_lp_count(grp_name, 1, LP_CONFIG_NM, s->anno, 0); int num_reps = codes_mapping_get_group_reps(grp_name); s->node_id = (mapping_rep_id * num_lps) + mapping_offset; @@ -223,17 +309,8 @@ static tw_stime torus_packet_event(char* category, tw_lpid final_dest_lp, uint64 tw_event * e_new; tw_stime xfer_to_nic_time; nodes_message * msg; - tw_lpid dest_nic_id; char* tmp_ptr; - int mapping_grp_id, mapping_rep_id, mapping_type_id, mapping_offset; - // TODO: be annotation-aware - codes_mapping_get_lp_info(final_dest_lp, grp_name, &mapping_grp_id, NULL, - &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); - codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1, mapping_rep_id, - mapping_offset, &dest_nic_id); - - /* TODO: Should send the packets in correct sequence. Currently the last packet is being sent first due to codes_local_latency offset. */ xfer_to_nic_time = g_tw_lookahead + codes_local_latency(sender); /* Throws an error of found last KP time > current event time otherwise */ //e_new = tw_event_new(local_nic_id, xfer_to_nic_time+offset, sender); //msg = tw_event_data(e_new); @@ -241,7 +318,6 @@ static tw_stime torus_packet_event(char* category, tw_lpid final_dest_lp, uint64 sender, TORUS, (void**)&msg, (void**)&tmp_ptr); strcpy(msg->category, category); msg->final_dest_gid = final_dest_lp; - msg->dest_lp = dest_nic_id; msg->sender_svr= src_lp; msg->packet_size = packet_size; msg->remote_event_size_bytes = 0; @@ -250,13 +326,6 @@ static tw_stime torus_packet_event(char* category, tw_lpid final_dest_lp, uint64 msg->is_pull = is_pull; msg->pull_size = pull_size; - num_chunks = msg->packet_size/chunk_size; - - if(msg->packet_size % chunk_size) - { - num_chunks++; - } - if(is_last_pckt) /* Its the last packet so pass in remote event information*/ { if(remote_event_size > 0) @@ -282,90 +351,101 @@ static void torus_init( nodes_state * s, tw_lp * lp ) { int i, j; - int dim_N[ n_dims + 1 ]; + char anno[MAX_NAME_LENGTH]; - // TODO: be annotation-aware - codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); + codes_mapping_get_lp_info(lp->gid, grp_name, &mapping_grp_id, NULL, &mapping_type_id, anno, &mapping_rep_id, &mapping_offset); + if (anno[0] == '\0'){ + s->anno = NULL; + s->params = &all_params[num_params-1]; + } + else{ + s->anno = anno; + int id = configuration_get_annotation_index(anno, anno_map); + s->params = &all_params[id]; + } + int dim_N[ s->params->n_dims + 1 ]; - dim_N[ 0 ]=mapping_rep_id + mapping_offset; + // shorthand + const torus_param *p = s->params; - s->neighbour_minus_lpID = (int*)malloc(n_dims * sizeof(int)); - s->neighbour_plus_lpID = (int*)malloc(n_dims * sizeof(int)); - s->dim_position = (int*)malloc(n_dims * sizeof(int)); - s->buffer = (int**)malloc(2*n_dims * sizeof(int*)); - s->next_link_available_time = (tw_stime**)malloc(2*n_dims * sizeof(tw_stime*)); - s->next_credit_available_time = (tw_stime**)malloc(2*n_dims * sizeof(tw_stime*)); - s->next_flit_generate_time = (tw_stime**)malloc(2*n_dims*sizeof(tw_stime*)); + dim_N[ 0 ]=mapping_rep_id + mapping_offset; - for(i=0; i < 2*n_dims; i++) + s->neighbour_minus_lpID = (int*)malloc(p->n_dims * sizeof(int)); + s->neighbour_plus_lpID = (int*)malloc(p->n_dims * sizeof(int)); + s->dim_position = (int*)malloc(p->n_dims * sizeof(int)); + s->buffer = (int**)malloc(2*p->n_dims * sizeof(int*)); + s->next_link_available_time = + (tw_stime**)malloc(2*p->n_dims * sizeof(tw_stime*)); + s->next_credit_available_time = + (tw_stime**)malloc(2*p->n_dims * sizeof(tw_stime*)); + s->next_flit_generate_time = + (tw_stime**)malloc(2*p->n_dims*sizeof(tw_stime*)); + + for(i=0; i < 2*p->n_dims; i++) { - s->buffer[i] = (int*)malloc(num_vc * sizeof(int)); - s->next_link_available_time[i] = (tw_stime*)malloc(num_vc * sizeof(tw_stime)); - s->next_credit_available_time[i] = (tw_stime*)malloc(num_vc * sizeof(tw_stime)); - s->next_flit_generate_time[i] = (tw_stime*)malloc(num_vc * sizeof(tw_stime)); + s->buffer[i] = (int*)malloc(p->num_vc * sizeof(int)); + s->next_link_available_time[i] = + (tw_stime*)malloc(p->num_vc * sizeof(tw_stime)); + s->next_credit_available_time[i] = + (tw_stime*)malloc(p->num_vc * sizeof(tw_stime)); + s->next_flit_generate_time[i] = + (tw_stime*)malloc(p->num_vc * sizeof(tw_stime)); } //printf("\n LP ID %d ", (int)lp->gid); // calculate my torus co-ordinates - for ( i=0; i < n_dims; i++ ) + for ( i=0; i < p->n_dims; i++ ) { - s->dim_position[ i ] = dim_N[ i ]%dim_length[ i ]; + s->dim_position[i] = dim_N[i]%p->dim_length[i]; //printf(" dim position %d ", s->dim_position[i]); - dim_N[ i + 1 ] = ( dim_N[ i ] - s->dim_position[ i ] )/dim_length[ i ]; - - half_length[ i ] = dim_length[ i ] / 2; + dim_N[i + 1] = ( dim_N[i] - s->dim_position[i] )/p->dim_length[i]; } //printf("\n"); - factor[ 0 ] = 1; - for ( i=1; i < n_dims; i++ ) - { - factor[ i ] = 1; - for ( j = 0; j < i; j++ ) - factor[ i ] *= dim_length[ j ]; - } - int temp_dim_pos[ n_dims ]; - for ( i = 0; i < n_dims; i++ ) + int temp_dim_pos[ p->n_dims ]; + for ( i = 0; i < p->n_dims; i++ ) temp_dim_pos[ i ] = s->dim_position[ i ]; tw_lpid neighbor_id; // calculate minus neighbour's lpID - for ( j = 0; j < n_dims; j++ ) + for ( j = 0; j < p->n_dims; j++ ) { - temp_dim_pos[ j ] = (s->dim_position[ j ] -1 + dim_length[ j ]) % dim_length[ j ]; + temp_dim_pos[ j ] = (s->dim_position[ j ] -1 + p->dim_length[ j ]) % + p->dim_length[ j ]; s->neighbour_minus_lpID[ j ] = 0; - for ( i = 0; i < n_dims; i++ ) - s->neighbour_minus_lpID[ j ] += factor[ i ] * temp_dim_pos[ i ]; + for ( i = 0; i < p->n_dims; i++ ) + s->neighbour_minus_lpID[ j ] += p->factor[ i ] * temp_dim_pos[ i ]; - codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1, + codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, s->anno, 0, s->neighbour_minus_lpID[ j ], 0, &neighbor_id); - //printf("\n neighbor %d lp id %d ", (int)s->neighbour_minus_lpID[ j ], (int)neighbor_id); + //printf("\n minus neighbor %d lp id %d ", (int)s->neighbour_minus_lpID[ j ], (int)neighbor_id); temp_dim_pos[ j ] = s->dim_position[ j ]; } // calculate plus neighbour's lpID - for ( j = 0; j < n_dims; j++ ) + for ( j = 0; j < p->n_dims; j++ ) { - temp_dim_pos[ j ] = ( s->dim_position[ j ] + 1 + dim_length[ j ]) % dim_length[ j ]; + temp_dim_pos[ j ] = ( s->dim_position[ j ] + 1 + p->dim_length[ j ]) % + p->dim_length[ j ]; s->neighbour_plus_lpID[ j ] = 0; - for ( i = 0; i < n_dims; i++ ) - s->neighbour_plus_lpID[ j ] += factor[ i ] * temp_dim_pos[ i ]; + for ( i = 0; i < s->params->n_dims; i++ ) + s->neighbour_plus_lpID[ j ] += p->factor[ i ] * temp_dim_pos[ i ]; - codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, NULL, 1, + codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, s->anno, 0, s->neighbour_plus_lpID[ j ], 0, &neighbor_id); - //printf("\n neighbor %d lp id %d ", (int)s->neighbour_plus_lpID[ j ], (int)neighbor_id); + //printf("\n plus neighbor %d lp id %d ", (int)s->neighbour_plus_lpID[ j ], (int)neighbor_id); temp_dim_pos[ j ] = s->dim_position[ j ]; } //printf("\n"); - for( j=0; j < 2 * n_dims; j++ ) + for( j=0; j < 2 * p->n_dims; j++ ) { - for( i = 0; i < num_vc; i++ ) + for( i = 0; i < p->num_vc; i++ ) { s->buffer[ j ][ i ] = 0; s->next_link_available_time[ j ][ i ] = 0.0; @@ -433,7 +513,7 @@ static void send_remote_event(nodes_state * s, tw_event* e; tw_stime ts; nodes_message * m; - ts = (1/link_bandwidth) * msg->remote_event_size_bytes; + ts = (1/s->params->link_bandwidth) * msg->remote_event_size_bytes; e = codes_event_new(s->origin_svr, ts, lp); m = tw_event_data(e); char* tmp_ptr = (char*)msg; @@ -654,8 +734,8 @@ static void dimension_order_routing( nodes_state * s, int * dim, int * dir ) { - int dim_N[n_dims], - dest[n_dims], + int dim_N[s->params->n_dims], + dest[s->params->n_dims], i, dest_id=0; @@ -664,36 +744,38 @@ static void dimension_order_routing( nodes_state * s, dim_N[ 0 ]=mapping_rep_id + mapping_offset; // find destination dimensions using destination LP ID - for ( i = 0; i < n_dims; i++ ) + for ( i = 0; i < s->params->n_dims; i++ ) { - dest[ i ] = dim_N[ i ] % dim_length[ i ]; - dim_N[ i + 1 ] = ( dim_N[ i ] - dest[ i ] ) / dim_length[ i ]; + dest[ i ] = dim_N[ i ] % s->params->dim_length[ i ]; + dim_N[ i + 1 ] = ( dim_N[ i ] - dest[ i ] ) / s->params->dim_length[ i ]; } - for( i = 0; i < n_dims; i++ ) + for( i = 0; i < s->params->n_dims; i++ ) { - if ( s->dim_position[ i ] - dest[ i ] > half_length[ i ] ) + if ( s->dim_position[ i ] - dest[ i ] > s->params->half_length[ i ] ) { dest_id = s->neighbour_plus_lpID[ i ]; *dim = i; *dir = 1; break; } - if ( s->dim_position[ i ] - dest[ i ] < -half_length[ i ] ) + if ( s->dim_position[ i ] - dest[ i ] < -s->params->half_length[ i ] ) { dest_id = s->neighbour_minus_lpID[ i ]; *dim = i; *dir = 0; break; } - if ( ( s->dim_position[ i ] - dest[ i ] <= half_length[ i ] ) && ( s->dim_position[ i ] - dest[ i ] > 0 ) ) + if ( ( s->dim_position[i] - dest[i] <= s->params->half_length[i] ) && + ( s->dim_position[ i ] - dest[ i ] > 0 ) ) { dest_id = s->neighbour_minus_lpID[ i ]; *dim = i; *dir = 0; break; } - if (( s->dim_position[ i ] - dest[ i ] >= -half_length[ i ] ) && ( s->dim_position[ i ] - dest[ i ] < 0) ) + if (( s->dim_position[i] - dest[i] >= -s->params->half_length[i] ) && + ( s->dim_position[ i ] - dest[ i ] < 0) ) { dest_id = s->neighbour_plus_lpID[ i ]; *dim = i; @@ -722,7 +804,17 @@ static void packet_generate( nodes_state * s, tw_event * e_h; nodes_message *m; - tw_lpid dst_lp = msg->dest_lp; + int mapping_grp_id, mapping_rep_id, mapping_type_id, mapping_offset; + tw_lpid dst_lp; + // TODO: be annotation-aware + codes_mapping_get_lp_info(msg->final_dest_gid, grp_name, &mapping_grp_id, NULL, + &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); + codes_mapping_get_lp_id(grp_name, LP_CONFIG_NM, s->anno, 0, mapping_rep_id, + mapping_offset, &dst_lp); + // dest_lp gets included to other required msgs through memcpys, so just + // set here + msg->dest_lp = dst_lp; + dimension_order_routing( s, &dst_lp, &tmp_dim, &tmp_dir ); msg->saved_src_dim = tmp_dim; @@ -733,6 +825,10 @@ static void packet_generate( nodes_state * s, msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter; msg->my_N_hop = 0; + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; + s->packet_counter++; @@ -740,7 +836,7 @@ static void packet_generate( nodes_state * s, printf("\n packet generated %lld at lp %d dest %d final dest %d", msg->packet_ID, (int)lp->gid, (int)dst_lp, (int)msg->dest_lp); for(j = 0; j < num_chunks; j++) { - if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < buffer_size) + if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < s->params->buffer_size) { ts = j + tw_rand_exponential(lp->rng, MEAN_INTERVAL/200); //s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0] = max(s->next_flit_generate_time[(2*tmp_dim) + tmp_dir][0], tw_now(lp)); @@ -790,7 +886,7 @@ static void packet_generate( nodes_state * s, stat = model_net_find_stats(msg->category, s->torus_stats_array); stat->send_count++; stat->send_bytes += msg->packet_size; - stat->send_time += (1/link_bandwidth) * msg->packet_size; + stat->send_time += (1/s->params->link_bandwidth) * msg->packet_size; /* record the maximum ROSS event size */ if(stat->max_event_size < total_event_size) stat->max_event_size = total_event_size; @@ -802,8 +898,7 @@ static void credit_send( nodes_state * s, nodes_message * msg) { #if DEBUG -//if(lp->gid == TRACK_LP) -// printf("\n (%lf) sending credit tmp_dir %d tmp_dim %d %lf ", tw_now(lp), msg->source_direction, msg->source_dim, credit_delay ); + //printf("\n (%lf) sending credit tmp_dir %d tmp_dim %d %lf ", tw_now(lp), msg->source_direction, msg->source_dim, s->params->credit_delay ); #endif bf->c1 = 0; tw_event * buf_e; @@ -814,7 +909,8 @@ static void credit_send( nodes_state * s, msg->saved_available_time = s->next_credit_available_time[(2 * src_dim) + src_dir][0]; s->next_credit_available_time[(2 * src_dim) + src_dir][0] = maxd(s->next_credit_available_time[(2 * src_dim) + src_dir][0], tw_now(lp)); - ts = credit_delay + tw_rand_exponential(lp->rng, credit_delay/1000); + ts = s->params->credit_delay + + tw_rand_exponential(lp->rng, s->params->credit_delay/1000); s->next_credit_available_time[(2 * src_dim) + src_dir][0] += ts; //buf_e = tw_event_new( msg->sender_lp, s->next_credit_available_time[(2 * src_dim) + src_dir][0] - tw_now(lp), lp); @@ -844,12 +940,13 @@ static void packet_send( nodes_state * s, tw_lpid dst_lp = msg->dest_lp; dimension_order_routing( s, &dst_lp, &tmp_dim, &tmp_dir ); - if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < buffer_size) + if(s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ] < s->params->buffer_size) { bf->c2 = 1; msg->saved_src_dir = tmp_dir; msg->saved_src_dim = tmp_dim; - ts = tw_rand_exponential( lp->rng, ( double )head_delay/200 )+ head_delay; + ts = tw_rand_exponential( lp->rng, s->params->head_delay/200.0 ) + + s->params->head_delay; // For reverse computation msg->saved_available_time = s->next_link_available_time[tmp_dir + ( tmp_dim * 2 )][0]; @@ -884,6 +981,11 @@ static void packet_send( nodes_state * s, s->buffer[ tmp_dir + ( tmp_dim * 2 ) ][ 0 ]++; + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + + if(msg->packet_size % s->params->chunk_size) + num_chunks++; + if(msg->chunk_id == num_chunks - 1) { bf->c1 = 1; @@ -893,7 +995,7 @@ static void packet_send( nodes_state * s, tw_event* e_new; nodes_message* m_new; void* local_event; - ts = (1/link_bandwidth) * msg->local_event_size_bytes; + ts = (1/s->params->link_bandwidth) * msg->local_event_size_bytes; e_new = tw_event_new(msg->sender_svr, ts, lp); m_new = tw_event_data(e_new); //local_event = (char*)msg; @@ -935,6 +1037,10 @@ static void packet_arrive( nodes_state * s, printf("\n packet arrived at lp %d final dest %d ", (int)lp->gid, (int)msg->dest_lp); if( lp->gid == msg->dest_lp ) { + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; + if( msg->chunk_id == num_chunks - 1 ) { bf->c2 = 1; @@ -956,12 +1062,11 @@ static void packet_arrive( nodes_state * s, // Trigger an event on receiving server if(msg->remote_event_size_bytes) { - ts = (1/link_bandwidth) * msg->remote_event_size_bytes; void *tmp_ptr = model_net_method_get_edata(TORUS, msg); if (msg->is_pull){ int net_id = model_net_get_id(LP_METHOD_NM); model_net_event(net_id, msg->category, msg->sender_svr, - msg->pull_size, ts, msg->remote_event_size_bytes, + msg->pull_size, 0.0, msg->remote_event_size_bytes, tmp_ptr, 0, NULL, lp); } else{ @@ -1017,7 +1122,12 @@ final( nodes_state * s, tw_lp * lp ) free(s->next_link_available_time); free(s->next_credit_available_time); free(s->next_flit_generate_time); - free(s->buffer); + // since all LPs are sharing params, just let them leak for now + // TODO: add a post-sim "cleanup" function? + //free(s->buffer); + //free(s->params->dim_length); + //free(s->params->factor); + //free(s->params->half_length); } /* increments the buffer count after a credit arrives from the remote compute node */ @@ -1038,6 +1148,10 @@ static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw //saved_dim = msg->saved_src_dim; //saved_dir = msg->saved_src_dir; + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; + //s->next_flit_generate_time[(saved_dim * 2) + saved_dir][0] = msg->saved_available_time; for(i=0; i < num_chunks; i++) tw_rand_reverse_unif(lp->rng); @@ -1045,7 +1159,7 @@ static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw stat = model_net_find_stats(msg->category, s->torus_stats_array); stat->send_count--; stat->send_bytes -= msg->packet_size; - stat->send_time -= (1/link_bandwidth) * msg->packet_size; + stat->send_time -= (1/s->params->link_bandwidth) * msg->packet_size; } break; @@ -1055,6 +1169,9 @@ static void node_rc_handler(nodes_state * s, tw_bf * bf, nodes_message * msg, tw tw_rand_reverse_unif(lp->rng); int next_dim = msg->source_dim; int next_dir = msg->source_direction; + uint64_t num_chunks = msg->packet_size/s->params->chunk_size; + if(msg->packet_size % s->params->chunk_size) + num_chunks++; s->next_credit_available_time[next_dir + ( next_dim * 2 )][0] = msg->saved_available_time; if(bf->c2) @@ -1205,7 +1322,7 @@ static tw_lpid torus_find_local_device(tw_lp *sender) /* data structure for torus statistics */ struct model_net_method torus_method = { - .mn_setup = torus_setup, + .mn_configure = torus_configure, .model_net_method_packet_event = torus_packet_event, .model_net_method_packet_event_rc = torus_packet_event_rc, .mn_get_lp_type = torus_get_lp_type, diff --git a/tests/concurrent-msg-recv.c b/tests/concurrent-msg-recv.c index 452611b3b2b8f180ebb70ec0e59e88b77a54eb57..a7ea6ea1688ff7c647dc5e95f627c3094de60054 100644 --- a/tests/concurrent-msg-recv.c +++ b/tests/concurrent-msg-recv.c @@ -475,13 +475,6 @@ int main(int argc, char *argv[]) return 1; } - /* Setup the model-net parameters specified in the global config object, - * returned are the identifier for the network type */ - net_ids = model_net_set_params(&num_nets); - assert(num_nets==1); - net_id = *net_ids; - free(net_ids); - /* currently restrict to simplenet, as other networks are trickier to * setup. TODO: handle other networks properly */ if(net_id != SIMPLENET) { @@ -494,6 +487,13 @@ int main(int argc, char *argv[]) codes_mapping_setup(); + /* Setup the model-net parameters specified in the global config object, + * returned are the identifier for the network type */ + net_ids = model_net_configure(&num_nets); + assert(num_nets==1); + net_id = *net_ids; + free(net_ids); + tw_run(); tw_end(); diff --git a/tests/conf/modelnet-test-simplewan.conf b/tests/conf/modelnet-test-simplewan.conf index f3e16aaa866f26bee78490b2cf72cc5dabc6b791..9575698234ed09abbfa95f866808908426b8d7f7 100644 --- a/tests/conf/modelnet-test-simplewan.conf +++ b/tests/conf/modelnet-test-simplewan.conf @@ -11,7 +11,7 @@ PARAMS { message_size="256"; packet_size="1024"; - modelnet="simplewan"; + modelnet_order=("simplewan"); # scheduler options modelnet_scheduler="fcfs"; # modelnet_scheduler="round-robin"; diff --git a/tests/modelnet-p2p-bw.c b/tests/modelnet-p2p-bw.c index e8fee957fc109414ebcf998eea64972480a84dda..3ad5a34c25a6ced8dc69bff36155db4b126abc9d 100644 --- a/tests/modelnet-p2p-bw.c +++ b/tests/modelnet-p2p-bw.c @@ -148,14 +148,15 @@ int main( MPI_Comm_size(MPI_COMM_WORLD, &nprocs); configuration_load(argv[2], MPI_COMM_WORLD, &config); - net_ids = model_net_set_params(&num_nets); - assert(num_nets==1); - net_id = *net_ids; - free(net_ids); svr_add_lp_type(); + model_net_register(); codes_mapping_setup(); + net_ids = model_net_configure(&num_nets); + assert(num_nets==1); + net_id = *net_ids; + free(net_ids); num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "server", NULL, 1); assert(num_servers == 2); diff --git a/tests/modelnet-simplewan-test.c b/tests/modelnet-simplewan-test.c index edce1c1fbc371de172c81340af9fbc910e1711e5..6f5fa133115418ba9df3bd849a4e5fbc796dc6bd 100644 --- a/tests/modelnet-simplewan-test.c +++ b/tests/modelnet-simplewan-test.c @@ -160,13 +160,15 @@ int main( MPI_Comm_size(MPI_COMM_WORLD, &nprocs); configuration_load(argv[2], MPI_COMM_WORLD, &config); - net_ids = model_net_set_params(&num_nets); - assert(num_nets==1); - net_id = *net_ids; - free(net_ids); svr_add_lp_type(); + model_net_register(); codes_mapping_setup(); + + net_ids = model_net_configure(&num_nets); + assert(num_nets==1); + net_id = *net_ids; + free(net_ids); num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "server", NULL, 1); diff --git a/tests/modelnet-test-collective.c b/tests/modelnet-test-collective.c index 4423110667d7e9391adffd9638e3a0eeebccd4d1..02120dce20cdbf5300c5d562fcc346d69ee65722 100644 --- a/tests/modelnet-test-collective.c +++ b/tests/modelnet-test-collective.c @@ -160,13 +160,14 @@ int main( MPI_Comm_size(MPI_COMM_WORLD, &nprocs); configuration_load(argv[2], MPI_COMM_WORLD, &config); - net_ids = model_net_set_params(&num_nets); - assert(num_nets==1); - net_id = *net_ids; - free(net_ids); svr_add_lp_type(); codes_mapping_setup(); + + net_ids = model_net_configure(&num_nets); + assert(num_nets==1); + net_id = *net_ids; + free(net_ids); num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "server", NULL, 1); diff --git a/tests/modelnet-test.c b/tests/modelnet-test.c index 017e912de20dc73ac5715dcc0955906ee45e31bb..6779f44132c3133a558e0ce4e4ed7e0a88dc7fd8 100644 --- a/tests/modelnet-test.c +++ b/tests/modelnet-test.c @@ -171,14 +171,17 @@ int main( MPI_Comm_size(MPI_COMM_WORLD, &nprocs); configuration_load(argv[2], MPI_COMM_WORLD, &config); - net_ids = model_net_set_params(&num_nets); - assert(num_nets==1); - net_id = *net_ids; - free(net_ids); + + model_net_register(); svr_add_lp_type(); codes_mapping_setup(); + net_ids = model_net_configure(&num_nets); + assert(num_nets==1); + net_id = *net_ids; + free(net_ids); + num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "server", NULL, 1); if(net_id == DRAGONFLY)