Commit 4e19d940 authored by Jonathan Jenkins's avatar Jonathan Jenkins

config overhaul + annotation-aware mapping

All configuration now proceeds on a per-LP level, and requires separate
registration and configuration calls, as seen in the test programs.
model_net_set_params is no longer used, and is replaced by model_net_register
and model_net_configure. The dragonfly network, having two LP types bundled in
the same code-path, is special-cased in the registration code.

LP-mapping in model-net now has the following defaults:
- counts via codes_mapping_get_lp_count are now with respect to the calling
  network LP's annotation.
- when looking up network LPs via
  codes_mapping_get_lp_info/codes_mapping_get_lp_id, the annotation of the
  calling network LP is used. Hence, routing now occurs only between LPs of the
  same annotation. If the destination LP's group specified by model_net_*event
  does not contain a modelnet LP with the same annotation as the modelnet LP in
  the sender's group, then an error will occur (in codes_mapping).

Known Issues:
- modelnet users currently cannot specify which modelnet LP to use in the case
  of multiple modelnet LPs in the sender's group. This will be fixed in future
  commits after a consensus is reached on the best way to expose this
  information.
parent c65c84a1
...@@ -24,6 +24,12 @@ ...@@ -24,6 +24,12 @@
extern int model_net_base_magic; extern int model_net_base_magic;
// register the networks with ROSS, given the array of flags, one for each
// network type
void model_net_base_register(int *do_config_nets);
// configure the base LP type, setting up general parameters
void model_net_base_configure();
// Construct a model-net-specific event, analagous to a tw_event_new and // Construct a model-net-specific event, analagous to a tw_event_new and
// codes_event_new. The difference here is that we return pointers to // codes_event_new. The difference here is that we return pointers to
// both the message data (to be cast into the appropriate type) and the // both the message data (to be cast into the appropriate type) and the
...@@ -49,7 +55,7 @@ void * model_net_method_get_edata(int net_id, void * msg); ...@@ -49,7 +55,7 @@ void * model_net_method_get_edata(int net_id, void * msg);
/// use them /// use them
// init method // init method
void model_net_base_init(int id_count, int *ids); void model_net_base_init(int *do_config_nets);
enum model_net_base_event_type { enum model_net_base_event_type {
MN_BASE_NEW_MSG, MN_BASE_NEW_MSG,
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
struct model_net_method struct model_net_method
{ {
uint64_t packet_size; /* packet size */ uint64_t packet_size; /* packet size */
void (*mn_setup)(const void* net_params); /* For initializing the network */ void (*mn_configure)(); /* For initializing the network */
tw_stime (*model_net_method_packet_event)( tw_stime (*model_net_method_packet_event)(
char* category, char* category,
tw_lpid final_dest_lp, tw_lpid final_dest_lp,
......
...@@ -30,10 +30,6 @@ enum sched_type { ...@@ -30,10 +30,6 @@ enum sched_type {
extern char * sched_names[]; extern char * sched_names[];
/// global for scheduler
/// TODO: move away from using the global for when we have multiple networks
extern enum sched_type mn_sched_type;
/// scheduler decls /// scheduler decls
typedef struct model_net_sched_s model_net_sched; typedef struct model_net_sched_s model_net_sched;
......
...@@ -24,7 +24,8 @@ ...@@ -24,7 +24,8 @@
* messages as part of an event and expects FCFS ordering. A proper fix which * messages as part of an event and expects FCFS ordering. A proper fix which
* involves model-net LP-level scheduling of requests is ideal, but not * involves model-net LP-level scheduling of requests is ideal, but not
* feasible for now (would basically have to redesign model-net), so expose * feasible for now (would basically have to redesign model-net), so expose
* explicit start-sequence and stop-sequence markers as a workaround */ * explicit start-sequence and stop-sequence markers as a workaround
*/
extern int in_sequence; extern int in_sequence;
extern tw_stime mn_msg_offset; extern tw_stime mn_msg_offset;
#define MN_START_SEQ() do {\ #define MN_START_SEQ() do {\
...@@ -36,11 +37,6 @@ extern tw_stime mn_msg_offset; ...@@ -36,11 +37,6 @@ extern tw_stime mn_msg_offset;
} while (0) } while (0)
typedef struct simplenet_param simplenet_param;
typedef struct simplewan_param simplewan_param;
typedef struct dragonfly_param dragonfly_param;
typedef struct torus_param torus_param;
typedef struct loggp_param loggp_param;
typedef struct mn_stats mn_stats; typedef struct mn_stats mn_stats;
// use the X-macro to get types and names rolled up into one structure // use the X-macro to get types and names rolled up into one structure
...@@ -93,53 +89,18 @@ struct mn_stats ...@@ -93,53 +89,18 @@ struct mn_stats
long max_event_size; long max_event_size;
}; };
/* structs for initializing a network/ specifying network parameters */ /* Registers all model-net LPs in ROSS. Should be called after
struct loggp_param * configuration_load, but before codes_mapping_setup */
{ void model_net_register();
char* net_config_file; /* file with loggp parameter table */
};
/* structs for initializing a network/ specifying network parameters */
struct simplenet_param
{
double net_startup_ns; /*simplenet startup cost*/
double net_bw_mbps; /*Link bandwidth per byte*/
int num_nics;
};
struct simplewan_param
{
char bw_filename[MAX_NAME_LENGTH];
char startup_filename[MAX_NAME_LENGTH];
};
struct dragonfly_param /* Configures all model-net LPs based on the CODES configuration, and returns
{ * ids to address the different types by.
int num_routers; /*Number of routers in a group*/ *
double local_bandwidth;/* bandwidth of the router-router channels within a group */ * id_count - the output number of networks
double global_bandwidth;/* bandwidth of the inter-group router connections */ *
double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */ * return - the set of network IDs, indexed in the order given by the
int num_vcs; /* number of virtual channels */ * modelnet_order configuration parameter */
int local_vc_size; /* buffer size of the router-router channels */ int* model_net_configure(int *id_count);
int global_vc_size; /* buffer size of the global channels */
int cn_vc_size; /* buffer size of the compute node channels */
short routing; /* minimal or non-minimal routing */
short chunk_size; /* full-sized packets are broken into smaller chunks.*/
};
struct torus_param
{
int n_dims; /*Dimension of the torus network, 5-D, 7-D or any other*/
int* dim_length; /*Length of each torus dimension*/
double link_bandwidth;/* bandwidth for each torus link */
int buffer_size; /* number of buffer slots for each vc in flits*/
int num_vc; /* number of virtual channels for each torus link */
float mean_process;/* mean process time for each flit */
int chunk_size; /* chunk is the smallest unit--default set to 32 */
};
/* In general we need to figure out how to pass configuration information to
* the methods and we need to be able to calculate ross event message size.
*/
/* Initialize/configure the network(s) based on the CODES configuration. /* Initialize/configure the network(s) based on the CODES configuration.
* returns an array of the network ids, indexed in the order given by the * returns an array of the network ids, indexed in the order given by the
......
...@@ -40,34 +40,45 @@ static double maxd(double a, double b) { return a < b ? b : a; } ...@@ -40,34 +40,45 @@ static double maxd(double a, double b) { return a < b ? b : a; }
// arrival rate // arrival rate
static double MEAN_INTERVAL=200.0; static double MEAN_INTERVAL=200.0;
/* radix of a dragonfly router = number of global channels + number of // threshold for adaptive routing
* compute node channels + number of local router channels */ static int adaptive_threshold = 10;
static int radix=0;
/* configurable parameters, coming from the codes config file*/
/* number of virtual channels, number of routers comes from the
* config file, number of compute nodes, global channels and group
* is calculated from these configurable parameters */
static int num_vcs, num_routers, num_cn, num_global_channels, num_groups, chunk_size;
/* adaptive threshold is to bias the adaptive routing */
static int total_routers, adaptive_threshold = 10;
/* minimal and non-minimal packet counts for adaptive routing*/ /* minimal and non-minimal packet counts for adaptive routing*/
int minimal_count=0, nonmin_count=0; int minimal_count=0, nonmin_count=0;
/* configurable parameters, global channel, local channel and typedef struct dragonfly_param dragonfly_param;
* compute node bandwidth */ /* annotation-specific parameters (unannotated entry occurs at the
static double global_bandwidth, local_bandwidth, cn_bandwidth; * last index) */
static uint64_t num_params = 0;
/*configurable parameters, global virtual channel size, local static dragonfly_param * all_params = NULL;
* virtual channel size and compute node channel size */ static const config_anno_map_t * anno_map = NULL;
static int global_vc_size, local_vc_size, cn_vc_size;
/* global variables for codes mapping */ /* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH]; static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset; static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
struct dragonfly_param
{
// configuration parameters
int num_routers; /*Number of routers in a group*/
double local_bandwidth;/* bandwidth of the router-router channels within a group */
double global_bandwidth;/* bandwidth of the inter-group router connections */
double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
int num_vcs; /* number of virtual channels */
int local_vc_size; /* buffer size of the router-router channels */
int global_vc_size; /* buffer size of the global channels */
int cn_vc_size; /* buffer size of the compute node channels */
int routing; /* minimal or non-minimal routing */
int chunk_size; /* full-sized packets are broken into smaller chunks.*/
// derived parameters
int num_cn;
int num_groups;
int radix;
int total_routers;
int num_global_channels;
};
/* handles terminal and router events like packet generate/send/receive/buffer */ /* handles terminal and router events like packet generate/send/receive/buffer */
typedef enum event_t event_t; typedef enum event_t event_t;
...@@ -116,6 +127,9 @@ struct terminal_state ...@@ -116,6 +127,9 @@ struct terminal_state
/* to maintain a count of child nodes that have fanned in at the parent during the collective /* to maintain a count of child nodes that have fanned in at the parent during the collective
fan-in phase*/ fan-in phase*/
int num_fan_nodes; int num_fan_nodes;
const char * anno;
const dragonfly_param *params;
}; };
/* terminal event type (1-4) */ /* terminal event type (1-4) */
...@@ -169,13 +183,13 @@ struct router_state ...@@ -169,13 +183,13 @@ struct router_state
tw_stime* next_credit_available_time; tw_stime* next_credit_available_time;
int* vc_occupancy; int* vc_occupancy;
int* output_vc_state; int* output_vc_state;
const char * anno;
const dragonfly_param *params;
}; };
static short routing = MINIMAL; static short routing = MINIMAL;
static int head_delay;
static uint64_t num_chunks;
static tw_stime dragonfly_total_time = 0; static tw_stime dragonfly_total_time = 0;
static tw_stime dragonfly_max_latency = 0; static tw_stime dragonfly_max_latency = 0;
static tw_stime max_collective = 0; static tw_stime max_collective = 0;
...@@ -193,32 +207,106 @@ static int dragonfly_get_msg_sz(void) ...@@ -193,32 +207,106 @@ static int dragonfly_get_msg_sz(void)
return sizeof(terminal_message); return sizeof(terminal_message);
} }
static void dragonfly_read_config(const char * anno, dragonfly_param *params){
// shorthand
dragonfly_param *p = params;
/* setup the dragonfly model, initialize global parameters */ configuration_get_value_int(&config, "PARAMS", "num_routers", anno,
static void dragonfly_setup(const void* net_params) &p->num_routers);
{ if(p->num_routers <= 0) {
dragonfly_param* d_param = (dragonfly_param*)net_params; p->num_routers = 4;
fprintf(stderr, "Number of dimensions not specified, setting to %d\n",
num_vcs = d_param->num_vcs; p->num_routers);
num_routers = d_param->num_routers; }
num_cn = num_routers/2;
num_global_channels = num_routers/2; configuration_get_value_int(&config, "PARAMS", "num_vcs", anno,
num_groups = num_routers * num_cn + 1; &p->num_vcs);
if(p->num_vcs <= 0) {
global_bandwidth = d_param->global_bandwidth; p->num_vcs = 1;
local_bandwidth = d_param->local_bandwidth; fprintf(stderr, "Number of virtual channels not specified, setting to %d\n", p->num_vcs);
cn_bandwidth = d_param->cn_bandwidth; }
global_vc_size = d_param->global_vc_size; configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size);
local_vc_size = d_param->local_vc_size; if(!p->local_vc_size) {
cn_vc_size = d_param->cn_vc_size; p->local_vc_size = 1024;
routing = d_param->routing; fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size);
chunk_size = d_param->chunk_size; }
radix = num_vcs * (num_cn + num_global_channels + num_routers); configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size);
total_routers = num_groups * num_routers; if(!p->global_vc_size) {
lp_type_register("dragonfly_router", dragonfly_get_router_lp_type()); p->global_vc_size = 2048;
return; fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size);
}
configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size);
if(!p->cn_vc_size) {
p->cn_vc_size = 1024;
fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size);
}
configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
if(!p->chunk_size) {
p->chunk_size = 64;
fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
}
configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth);
if(!p->local_bandwidth) {
p->local_bandwidth = 5.25;
fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth);
}
configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth);
if(!p->global_bandwidth) {
p->global_bandwidth = 4.7;
fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth);
}
configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth);
if(!p->cn_bandwidth) {
p->cn_bandwidth = 5.25;
fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth);
}
char routing[MAX_NAME_LENGTH];
configuration_get_value(&config, "PARAMS", "routing", anno, routing,
MAX_NAME_LENGTH);
if(strcmp(routing, "minimal") == 0)
p->routing = 0;
else if(strcmp(routing, "nonminimal")==0 || strcmp(routing,"non-minimal")==0)
p->routing = 1;
else if (strcmp(routing, "adaptive") == 0)
p->routing = 2;
else
{
fprintf(stderr,
"No routing protocol specified, setting to minimal routing\n");
p->routing = 0;
}
// set the derived parameters
p->num_cn = p->num_routers/2;
p->num_global_channels = p->num_routers/2;
p->num_groups = p->num_routers * p->num_cn + 1;
p->radix = p->num_vcs *
(p->num_cn + p->num_global_channels + p->num_routers);
p->total_routers = p->num_groups * p->num_routers;
}
static void dragonfly_configure(){
anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
assert(anno_map);
num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
all_params = malloc(num_params * sizeof(*all_params));
for (uint64_t i = 0; i < anno_map->num_annos; i++){
const char * anno = anno_map->annotations[i];
dragonfly_read_config(anno, &all_params[i]);
}
if (anno_map->has_unanno_lp > 0){
dragonfly_read_config(NULL, &all_params[anno_map->num_annos]);
}
} }
/* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */ /* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */
...@@ -314,15 +402,8 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui ...@@ -314,15 +402,8 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui
tw_event * e_new; tw_event * e_new;
tw_stime xfer_to_nic_time; tw_stime xfer_to_nic_time;
terminal_message * msg; terminal_message * msg;
tw_lpid dest_nic_id;
char* tmp_ptr; char* tmp_ptr;
//TODO: be annotation-aware
codes_mapping_get_lp_info(final_dest_lp, lp_group_name, &mapping_grp_id,
NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1,
mapping_rep_id, mapping_offset, &dest_nic_id);
xfer_to_nic_time = codes_local_latency(sender); /* Throws an error of found last KP time > current event time otherwise when LPs of one type are placed together*/ xfer_to_nic_time = codes_local_latency(sender); /* Throws an error of found last KP time > current event time otherwise when LPs of one type are placed together*/
//printf("\n transfer in time %f %f ", xfer_to_nic_time+offset, tw_now(sender)); //printf("\n transfer in time %f %f ", xfer_to_nic_time+offset, tw_now(sender));
//e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender); //e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender);
...@@ -331,7 +412,6 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui ...@@ -331,7 +412,6 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui
sender, DRAGONFLY, (void**)&msg, (void**)&tmp_ptr); sender, DRAGONFLY, (void**)&msg, (void**)&tmp_ptr);
strcpy(msg->category, category); strcpy(msg->category, category);
msg->final_dest_gid = final_dest_lp; msg->final_dest_gid = final_dest_lp;
msg->dest_terminal_id = dest_nic_id;
msg->sender_lp=src_lp; msg->sender_lp=src_lp;
msg->packet_size = packet_size; msg->packet_size = packet_size;
msg->remote_event_size_bytes = 0; msg->remote_event_size_bytes = 0;
...@@ -372,17 +452,18 @@ static void dragonfly_packet_event_rc(tw_lp *sender) ...@@ -372,17 +452,18 @@ static void dragonfly_packet_event_rc(tw_lp *sender)
tw_lpid getRouterFromGroupID(int gid, tw_lpid getRouterFromGroupID(int gid,
router_state * r) router_state * r)
{ {
int group_begin = r->group_id * num_routers; const dragonfly_param *p = r->params;
int group_end = (r->group_id * num_routers) + num_routers-1; int group_begin = r->group_id * p->num_routers;
int offset = (gid * num_routers - group_begin) / num_routers; int group_end = (r->group_id * p->num_routers) + p->num_routers-1;
int offset = (gid * p->num_routers - group_begin) / p->num_routers;
if((gid * num_routers) < group_begin) if((gid * p->num_routers) < group_begin)
offset = (group_begin - gid * num_routers) / num_routers; // take absolute value offset = (group_begin - gid * p->num_routers) / p->num_routers; // take absolute value
int half_channel = num_global_channels / 2; int half_channel = p->num_global_channels / 2;
int index = (offset - 1)/(half_channel * num_routers); int index = (offset - 1)/(half_channel * p->num_routers);
offset=(offset - 1) % (half_channel * num_routers); offset=(offset - 1) % (half_channel * p->num_routers);
// If the destination router is in the same group // If the destination router is in the same group
tw_lpid router_id; tw_lpid router_id;
...@@ -405,30 +486,31 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -405,30 +486,31 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw
int dest=0, credit_delay=0, type = R_BUFFER; int dest=0, credit_delay=0, type = R_BUFFER;
int is_terminal = 0; int is_terminal = 0;
const dragonfly_param *p = s->params;
// Notify sender terminal about available buffer space // Notify sender terminal about available buffer space
if(msg->last_hop == TERMINAL) if(msg->last_hop == TERMINAL)
{ {
dest = msg->src_terminal_id; dest = msg->src_terminal_id;
//determine the time in ns to transfer the credit //determine the time in ns to transfer the credit
credit_delay = (1 / cn_bandwidth) * CREDIT_SIZE; credit_delay = (1 / p->cn_bandwidth) * CREDIT_SIZE;
type = T_BUFFER; type = T_BUFFER;
is_terminal = 1; is_terminal = 1;
} }
else if(msg->last_hop == GLOBAL) else if(msg->last_hop == GLOBAL)
{ {
dest = msg->intm_lp_id; dest = msg->intm_lp_id;
credit_delay = (1 / global_bandwidth) * CREDIT_SIZE; credit_delay = (1 / p->global_bandwidth) * CREDIT_SIZE;
} }
else if(msg->last_hop == LOCAL) else if(msg->last_hop == LOCAL)
{ {
dest = msg->intm_lp_id; dest = msg->intm_lp_id;
credit_delay = (1/local_bandwidth) * CREDIT_SIZE; credit_delay = (1/p->local_bandwidth) * CREDIT_SIZE;
} }
else else
printf("\n Invalid message type"); printf("\n Invalid message type");
// Assume it takes 0.1 ns of serialization latency for processing the credits in the queue // Assume it takes 0.1 ns of serialization latency for processing the credits in the queue
int output_port = msg->saved_vc / num_vcs; int output_port = msg->saved_vc / p->num_vcs;
msg->saved_available_time = s->next_credit_available_time[output_port]; msg->saved_available_time = s->next_credit_available_time[output_port];
s->next_credit_available_time[output_port] = maxd(tw_now(lp), s->next_credit_available_time[output_port]); s->next_credit_available_time[output_port] = maxd(tw_now(lp), s->next_credit_available_time[output_port]);
ts = credit_delay + 0.1 + tw_rand_exponential(lp->rng, (double)credit_delay/1000); ts = credit_delay + 0.1 + tw_rand_exponential(lp->rng, (double)credit_delay/1000);
...@@ -456,11 +538,22 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -456,11 +538,22 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw
/* generates packet at the current dragonfly compute node */ /* generates packet at the current dragonfly compute node */
void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
{ {
tw_lpid dest_terminal_id;
codes_mapping_get_lp_info(msg->final_dest_gid, lp_group_name, &mapping_grp_id,
NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, s->anno, 0,
mapping_rep_id, mapping_offset, &dest_terminal_id);
msg->dest_terminal_id = dest_terminal_id;
const dragonfly_param *p = s->params;
tw_stime ts; tw_stime ts;
tw_event *e; tw_event *e;
terminal_message *m; terminal_message *m;
int i, total_event_size; int i, total_event_size;
num_chunks = msg->packet_size / chunk_size; uint64_t num_chunks = msg->packet_size / p->chunk_size;
if (msg->packet_size % s->params->chunk_size)
num_chunks++;
msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter + tw_rand_integer(lp->rng, 0, lp->gid + g_tw_nlp * s->packet_counter); msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter + tw_rand_integer(lp->rng, 0, lp->gid + g_tw_nlp * s->packet_counter);
msg->travel_start_time = tw_now(lp); msg->travel_start_time = tw_now(lp);
msg->my_N_hop = 0; msg->my_N_hop = 0;
...@@ -470,9 +563,9 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ ...@@ -470,9 +563,9 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_
// msg->my_N_hop = 0; generating a packet, check if the input queue is available // msg->my_N_hop = 0; generating a packet, check if the input queue is available
ts = g_tw_lookahead + 0.1 + tw_rand_exponential(lp->rng, MEAN_INTERVAL/200); ts = g_tw_lookahead + 0.1 + tw_rand_exponential(lp->rng, MEAN_INTERVAL/200);
int chan = -1, j; int chan = -1, j;
for(j = 0; j < num_vcs; j++) for(j = 0; j < p->num_vcs; j++)
{ {
if(s->vc_occupancy[j] < cn_vc_size * num_chunks) if(s->vc_occupancy[j] < p->cn_vc_size * num_chunks)
{ {
chan=j; chan=j;
break; break;
...@@ -487,6 +580,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ ...@@ -487,6 +580,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_
e = model_net_method_event_new(lp->gid, i+ts, lp, DRAGONFLY, e = model_net_method_event_new(lp->gid, i+ts, lp, DRAGONFLY,
(void**)&m, &m_data); (void**)&m, &m_data);
memcpy(m, msg, sizeof(terminal_message)); memcpy(m, msg, sizeof(terminal_message));
m->dest_terminal_id = dest_terminal_id;
void * m_data_src = model_net_method_get_edata(DRAGONFLY, msg); void * m_data_src = model_net_method_get_edata(DRAGONFLY, msg);
if (msg->remote_event_size_bytes){ if (msg->remote_event_size_bytes){
memcpy(m_data, m_data_src, msg->remote_event_size_bytes); memcpy(m_data, m_data_src, msg->remote_event_size_bytes);
...@@ -523,7 +617,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ ...@@ -523,7 +617,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_
stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
stat->send_count++; stat->send_count++;
stat->send_bytes += msg->packet_size; stat->send_bytes += msg->packet_size;
stat->send_time += (1/cn_bandwidth) * msg->packet_size; stat->send_time += (1/p->cn_bandwidth) * msg->packet_size;
if(stat->max_event_size < total_event_size) if(stat->max_event_size < total_event_size)
stat->max_event_size = total_event_size; stat->max_event_size = total_event_size;
return; return;
...@@ -541,7 +635,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -541,7 +635,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
// Each packet is broken into chunks and then sent over the channel // Each packet is broken into chunks and then sent over the channel
msg->saved_available_time = s->terminal_available_time; msg->saved_available_time = s->terminal_available_time;
head_delay = (1/cn_bandwidth) * chunk_size; double head_delay = (1/s->params->cn_bandwidth) * s->params->chunk_size;
ts = head_delay + tw_rand_exponential(lp->rng, (double)head_delay/200); ts = head_delay + tw_rand_exponential(lp->rng, (double)head_delay/200);
s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp)); s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp));
s->terminal_available_time += ts; s->terminal_available_time += ts;
...@@ -554,6 +648,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -554,6 +648,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
// we are sending an event to the router, so no method_event here // we are sending an event to the router, so no method_event here
e = tw_event_new(router_id, s->terminal_available_time - tw_now(lp), lp); e = tw_event_new(router_id, s->terminal_available_time - tw_now(lp), lp);
uint64_t num_chunks = msg->packet_size/s->params->chunk_size;
if(msg->packet_size % s->params->chunk_size)
num_chunks++;
if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
printf("\n terminal %d packet %lld chunk %d being sent to router %d router id %d ", (int)lp->gid, (long long)msg->packet_ID, msg->chunk_id, (int)router_id, s->router_id); printf("\n terminal %d packet %lld chunk %d being sent to router %d router id %d ", (int)lp->gid, (long long)msg->packet_ID, msg->chunk_id, (int)router_id, s->router_id);
m = tw_event_data(e); m = tw_event_data(e);
...@@ -581,7 +679,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -581,7 +679,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
void* local_event = void* local_event =
(char*)model_net_method_get_edata(DRAGONFLY, msg) + (char*)model_net_method_get_edata(DRAGONFLY, msg) +
msg->remote_event_size_bytes; msg->remote_event_size_bytes;
ts = g_tw_lookahead + (1/cn_bandwidth) * msg->local_event_size_bytes; ts = g_tw_lookahead + (1/s->params->cn_bandwidth) * msg->local_event_size_bytes;
e_new = tw_event_new(msg->sender_lp, ts, lp); e_new = tw_event_new(msg->sender_lp, ts, lp);
m_new = tw_event_data(e_new); m_new = tw_event_data(e_new);
memcpy(m_new, local_event, msg->local_event_size_bytes); memcpy(m_new, local_event, msg->local_event_size_bytes);
...@@ -592,7 +690,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -592,7 +690,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
s->packet_counter++; s->packet_counter++;
s->vc_occupancy[vc]++; s->vc_occupancy[vc]++;
if(s->vc_occupancy[vc] >= (cn_vc_size * num_chunks)) if(s->vc_occupancy[vc] >= (s->params->cn_vc_size * num_chunks))
s->output_vc_state[vc] = VC_CREDIT; s->output_vc_state[vc] = VC_CREDIT;
return; return;
} }
...@@ -600,6 +698,9 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -600,6 +698,9 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
/* packet arrives at the destination terminal */ /* packet arrives at the destination terminal */
void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
{ {
uint64_t num_chunks = msg->packet_size / s->params->chunk_size;
if (msg->packet_size % s->params->chunk_size)
num_chunks++;
#if DEBUG #if DEBUG
if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
{ {
...@@ -644,7 +745,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) ...@@ -644,7 +745,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
if(msg->remote_event_size_bytes) if(msg->remote_event_size_bytes)
{ {
void * tmp_ptr = model_net_method_get_edata(DRAGONFLY, msg); void * tmp_ptr = model_net_method_get_edata(DRAGONFLY, msg);
ts = g_tw_lookahead + 0.1 + (1/cn_bandwidth) * msg->remote_event_size_bytes; ts = g_tw_lookahead + 0.1 + (1/s->params->cn_bandwidth) * msg->remote_event_size_bytes;
if (msg->is_pull){ if (msg->is_pull){
int net_id = model_net_get_id(LP_METHOD_NM); int net_id = model_net_get_id(LP_METHOD_NM);
model_net_event(net_id, msg->category, msg->sender_lp, model_net_event(net_id, msg->category, msg->sender_lp,
...@@ -660,7 +761,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) ...@@ -660,7 +761,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
} }
} }
int credit_delay = (1 / cn_bandwidth) * CREDIT_SIZE;