Commit c4c2afa8 authored by Misbah Mubarak's avatar Misbah Mubarak

Adding group level mapping in the dragonfly: multiple routers can now be...

Adding group level mapping in the dragonfly: multiple routers can now be specified in the config file
parent 5de10875
...@@ -14,6 +14,9 @@ typedef struct terminal_message terminal_message; ...@@ -14,6 +14,9 @@ typedef struct terminal_message terminal_message;
/* this message is used for both dragonfly compute nodes and routers */ /* this message is used for both dragonfly compute nodes and routers */
struct terminal_message struct terminal_message
{ {
/* magic number */
int magic;
/* flit travel start time*/ /* flit travel start time*/
tw_stime travel_start_time; tw_stime travel_start_time;
/* packet ID of the flit */ /* packet ID of the flit */
......
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
#include <ross.h> #include <ross.h>
#include "codes/codes_mapping.h" #include "codes/codes_mapping.h"
#include "codes/jenkins-hash.h"
#include "codes/codes.h" #include "codes/codes.h"
#include "codes/model-net.h" #include "codes/model-net.h"
#include "codes/model-net-method.h" #include "codes/model-net-method.h"
...@@ -58,6 +59,14 @@ static const config_anno_map_t * anno_map = NULL; ...@@ -58,6 +59,14 @@ static const config_anno_map_t * anno_map = NULL;
static char lp_group_name[MAX_NAME_LENGTH]; static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset; static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
/* router magic number */
int router_magic_num = 0;
/* terminal magic number */
int terminal_magic_num = 0;
/* number of routers in a mapping group */
static int num_routers_per_mgrp = 0;
struct dragonfly_param struct dragonfly_param
{ {
...@@ -93,8 +102,8 @@ struct terminal_state ...@@ -93,8 +102,8 @@ struct terminal_state
unsigned long long packet_counter; unsigned long long packet_counter;
// Dragonfly specific parameters // Dragonfly specific parameters
unsigned int router_id; tw_lpid router_id;
unsigned int terminal_id; tw_lpid terminal_id;
// Each terminal will have an input and output channel with the router // Each terminal will have an input and output channel with the router
int* vc_occupancy; // NUM_VC int* vc_occupancy; // NUM_VC
...@@ -230,7 +239,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ ...@@ -230,7 +239,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
configuration_get_value_int(&config, "PARAMS", "num_vcs", anno, configuration_get_value_int(&config, "PARAMS", "num_vcs", anno,
&p->num_vcs); &p->num_vcs);
if(p->num_vcs <= 0) { if(!p->num_vcs) {
p->num_vcs = 1; p->num_vcs = 1;
fprintf(stderr, "Number of virtual channels not specified, setting to %d\n", p->num_vcs); fprintf(stderr, "Number of virtual channels not specified, setting to %d\n", p->num_vcs);
} }
...@@ -304,10 +313,11 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){ ...@@ -304,10 +313,11 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
(p->num_cn + p->num_global_channels + p->num_routers); (p->num_cn + p->num_global_channels + p->num_routers);
p->total_routers = p->num_groups * p->num_routers; p->total_routers = p->num_groups * p->num_routers;
p->total_terminals = p->total_routers * p->num_cn; p->total_terminals = p->total_routers * p->num_cn;
printf("\n Total nodes %d routers %d groups %d radix %d ", p->num_cn * p->total_routers, printf("\n Total nodes %d routers %d groups %d radix %d num_vc %d ", p->num_cn * p->total_routers,
p->total_routers, p->total_routers,
p->num_groups, p->num_groups,
p->radix); p->radix,
p->num_vcs);
} }
static void dragonfly_configure(){ static void dragonfly_configure(){
...@@ -433,6 +443,7 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui ...@@ -433,6 +443,7 @@ static tw_stime dragonfly_packet_event(char* category, tw_lpid final_dest_lp, ui
msg->remote_event_size_bytes = 0; msg->remote_event_size_bytes = 0;
msg->local_event_size_bytes = 0; msg->local_event_size_bytes = 0;
msg->type = T_GENERATE; msg->type = T_GENERATE;
msg->magic = terminal_magic_num;
msg->is_pull = is_pull; msg->is_pull = is_pull;
msg->pull_size = pull_size; msg->pull_size = pull_size;
...@@ -500,6 +511,7 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -500,6 +511,7 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw
int dest=0, credit_delay=0, type = R_BUFFER; int dest=0, credit_delay=0, type = R_BUFFER;
int is_terminal = 0; int is_terminal = 0;
int found_magic = router_magic_num;
const dragonfly_param *p = s->params; const dragonfly_param *p = s->params;
int sender_radix; int sender_radix;
...@@ -512,6 +524,7 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -512,6 +524,7 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw
credit_delay = (1 / p->cn_bandwidth) * CREDIT_SIZE; credit_delay = (1 / p->cn_bandwidth) * CREDIT_SIZE;
type = T_BUFFER; type = T_BUFFER;
is_terminal = 1; is_terminal = 1;
found_magic = terminal_magic_num;
} }
else if(msg->last_hop == GLOBAL) else if(msg->last_hop == GLOBAL)
{ {
...@@ -551,6 +564,7 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -551,6 +564,7 @@ void router_credit_send(router_state * s, tw_bf * bf, terminal_message * msg, tw
buf_msg->origin_router_id = s->router_id; buf_msg->origin_router_id = s->router_id;
buf_msg->vc_index = msg->saved_vc; buf_msg->vc_index = msg->saved_vc;
buf_msg->type=type; buf_msg->type=type;
buf_msg->magic = found_magic;
buf_msg->last_hop = msg->last_hop; buf_msg->last_hop = msg->last_hop;
buf_msg->packet_ID=msg->packet_ID; buf_msg->packet_ID=msg->packet_ID;
...@@ -612,6 +626,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_ ...@@ -612,6 +626,7 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_
m->intm_group_id = -1; m->intm_group_id = -1;
m->saved_vc=0; m->saved_vc=0;
m->chunk_id = i; m->chunk_id = i;
m->magic = terminal_magic_num;
/* if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) /* if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
printf("\n packet generated %lld at terminal %d chunk id %d ", msg->packet_ID, (int)lp->gid, i); printf("\n packet generated %lld at terminal %d chunk id %d ", msg->packet_ID, (int)lp->gid, i);
...@@ -663,8 +678,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -663,8 +678,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
//TODO: be annotation-aware //TODO: be annotation-aware
codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
&mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", NULL, 1, codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", NULL, 1,
s->router_id, 0, &router_id); s->router_id/num_routers_per_mgrp, s->router_id % num_routers_per_mgrp, &router_id);
// we are sending an event to the router, so no method_event here // we are sending an event to the router, so no method_event here
e = tw_event_new(router_id, s->terminal_available_time - tw_now(lp), lp); e = tw_event_new(router_id, s->terminal_available_time - tw_now(lp), lp);
...@@ -680,6 +697,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * ...@@ -680,6 +697,7 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp *
memcpy(m+1, model_net_method_get_edata(DRAGONFLY, msg), memcpy(m+1, model_net_method_get_edata(DRAGONFLY, msg),
msg->remote_event_size_bytes); msg->remote_event_size_bytes);
} }
m->magic = router_magic_num;
m->origin_router_id = s->router_id; m->origin_router_id = s->router_id;
m->type = R_ARRIVE; m->type = R_ARRIVE;
m->src_terminal_id = lp->gid; m->src_terminal_id = lp->gid;
...@@ -812,6 +830,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) ...@@ -812,6 +830,7 @@ if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
// no method_event here - message going to router // no method_event here - message going to router
buf_e = tw_event_new(msg->intm_lp_id, s->next_credit_available_time - tw_now(lp), lp); buf_e = tw_event_new(msg->intm_lp_id, s->next_credit_available_time - tw_now(lp), lp);
buf_msg = tw_event_data(buf_e); buf_msg = tw_event_data(buf_e);
buf_msg->magic = router_magic_num;
buf_msg->vc_index = msg->saved_vc; buf_msg->vc_index = msg->saved_vc;
buf_msg->type=R_BUFFER; buf_msg->type=R_BUFFER;
buf_msg->packet_ID=msg->packet_ID; buf_msg->packet_ID=msg->packet_ID;
...@@ -826,6 +845,10 @@ void ...@@ -826,6 +845,10 @@ void
terminal_init( terminal_state * s, terminal_init( terminal_state * s,
tw_lp * lp ) tw_lp * lp )
{ {
uint32_t h1 = 0, h2 = 0;
bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
terminal_magic_num = h1 + h2;
int i; int i;
char anno[MAX_NAME_LENGTH]; char anno[MAX_NAME_LENGTH];
...@@ -859,6 +882,7 @@ terminal_init( terminal_state * s, ...@@ -859,6 +882,7 @@ terminal_init( terminal_state * s,
s->vc_occupancy[i]=0; s->vc_occupancy[i]=0;
s->output_vc_state[i]=VC_IDLE; s->output_vc_state[i]=VC_IDLE;
} }
// printf("\n Terminal ID %d Router ID %d ", s->terminal_id, s->router_id);
dragonfly_collective_init(s, lp); dragonfly_collective_init(s, lp);
return; return;
} }
...@@ -1153,6 +1177,7 @@ terminal_event( terminal_state * s, ...@@ -1153,6 +1177,7 @@ terminal_event( terminal_state * s,
terminal_message * msg, terminal_message * msg,
tw_lp * lp ) tw_lp * lp )
{ {
assert(msg->magic == terminal_magic_num);
*(int *)bf = (int)0; *(int *)bf = (int)0;
switch(msg->type) switch(msg->type)
{ {
...@@ -1255,7 +1280,7 @@ get_next_stop(router_state * s, ...@@ -1255,7 +1280,7 @@ get_next_stop(router_state * s,
codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
&mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
int local_router_id = (mapping_offset + mapping_rep_id); int local_router_id = mapping_offset + (mapping_rep_id * num_routers_per_mgrp);
bf->c2 = 0; bf->c2 = 0;
...@@ -1310,8 +1335,8 @@ get_next_stop(router_state * s, ...@@ -1310,8 +1335,8 @@ get_next_stop(router_state * s,
} }
} }
} }
codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", s->anno, 0, dest_lp, codes_mapping_get_lp_id(lp_group_name, "dragonfly_router", s->anno, 0, dest_lp/num_routers_per_mgrp,
0, &router_dest_id); dest_lp % num_routers_per_mgrp, &router_dest_id);
return router_dest_id; return router_dest_id;
} }
/* gets the output port corresponding to the next stop of the message */ /* gets the output port corresponding to the next stop of the message */
...@@ -1338,7 +1363,7 @@ get_output_port( router_state * s, ...@@ -1338,7 +1363,7 @@ get_output_port( router_state * s,
{ {
codes_mapping_get_lp_info(next_stop, lp_group_name, &mapping_grp_id, codes_mapping_get_lp_info(next_stop, lp_group_name, &mapping_grp_id,
NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset); NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
int local_router_id = mapping_rep_id + mapping_offset; int local_router_id = mapping_rep_id * num_routers_per_mgrp + mapping_offset;
int intm_grp_id = local_router_id / s->params->num_routers; int intm_grp_id = local_router_id / s->params->num_routers;
if(intm_grp_id != s->group_id) if(intm_grp_id != s->group_id)
...@@ -1565,19 +1590,12 @@ router_packet_send( router_state * s, ...@@ -1565,19 +1590,12 @@ router_packet_send( router_state * s,
{ {
s->cur_hist_num[output_chan]++; s->cur_hist_num[output_chan]++;
} }
/* if(lp->gid == TRACK)
printf("\n Sending packet from %d output chan %d VC occupancy %d stop %d dest term %d ",
lp->gid,
output_chan,
s->vc_occupancy[output_chan],
next_stop,
msg->dest_terminal_id);
*/
/* Determine the event type. If the packet has arrived at the final destination /* Determine the event type. If the packet has arrived at the final destination
router then it should arrive at the destination terminal next. */ router then it should arrive at the destination terminal next. */
if(next_stop == msg->dest_terminal_id) if(next_stop == msg->dest_terminal_id)
{ {
m->type = T_ARRIVE; m->type = T_ARRIVE;
m->magic = terminal_magic_num;
if(s->vc_occupancy[output_chan] >= s->params->cn_vc_size * num_chunks) if(s->vc_occupancy[output_chan] >= s->params->cn_vc_size * num_chunks)
s->output_vc_state[output_chan] = VC_CREDIT; s->output_vc_state[output_chan] = VC_CREDIT;
...@@ -1586,6 +1604,7 @@ router_packet_send( router_state * s, ...@@ -1586,6 +1604,7 @@ router_packet_send( router_state * s,
{ {
/* The packet has to be sent to another router */ /* The packet has to be sent to another router */
m->type = R_ARRIVE; m->type = R_ARRIVE;
m->magic = router_magic_num;
/* If this is a global channel then the buffer space is different */ /* If this is a global channel then the buffer space is different */
if( global ) if( global )
...@@ -1621,9 +1640,9 @@ router_packet_receive( router_state * s, ...@@ -1621,9 +1640,9 @@ router_packet_receive( router_state * s,
if(msg->packet_size % s->params->chunk_size) if(msg->packet_size % s->params->chunk_size)
num_chunks++; num_chunks++;
/* if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1) if(msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
printf("\n packet %lld chunk %d received at router %d ", msg->packet_ID, msg->chunk_id, (int)lp->gid); printf("\n packet %lld chunk %d received at router %d ", msg->packet_ID, msg->chunk_id, (int)lp->gid);
*/
router_credit_send(s, bf, msg, lp); router_credit_send(s, bf, msg, lp);
// router self message - no need for method_event // router self message - no need for method_event
...@@ -1638,10 +1657,16 @@ router_packet_receive( router_state * s, ...@@ -1638,10 +1657,16 @@ router_packet_receive( router_state * s,
/* sets up the router virtual channels, global channels, local channels, compute node channels */ /* sets up the router virtual channels, global channels, local channels, compute node channels */
void router_setup(router_state * r, tw_lp * lp) void router_setup(router_state * r, tw_lp * lp)
{ {
uint32_t h1 = 0, h2 = 0;
bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
router_magic_num = h1 + h2;
char anno[MAX_NAME_LENGTH]; char anno[MAX_NAME_LENGTH];
codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset); &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
num_routers_per_mgrp = codes_mapping_get_lp_count (lp_group_name, 1, "dragonfly_router",
NULL, 0);
if (anno[0] == '\0'){ if (anno[0] == '\0'){
r->anno = NULL; r->anno = NULL;
r->params = &all_params[num_params-1]; r->params = &all_params[num_params-1];
...@@ -1655,7 +1680,7 @@ void router_setup(router_state * r, tw_lp * lp) ...@@ -1655,7 +1680,7 @@ void router_setup(router_state * r, tw_lp * lp)
// shorthand // shorthand
const dragonfly_param *p = r->params; const dragonfly_param *p = r->params;
r->router_id=mapping_rep_id + mapping_offset; r->router_id=mapping_rep_id * num_routers_per_mgrp + mapping_offset;
r->group_id=r->router_id/p->num_routers; r->group_id=r->router_id/p->num_routers;
int i; int i;
...@@ -1733,6 +1758,7 @@ void router_buf_update(router_state * s, tw_bf * bf, terminal_message * msg, tw_ ...@@ -1733,6 +1758,7 @@ void router_buf_update(router_state * s, tw_bf * bf, terminal_message * msg, tw_
void router_event(router_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp) void router_event(router_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
{ {
assert(msg->magic == router_magic_num);
switch(msg->type) switch(msg->type)
{ {
case R_SEND: // Router has sent a packet to an intra-group router (local channel) case R_SEND: // Router has sent a packet to an intra-group router (local channel)
......
...@@ -2,10 +2,10 @@ LPGROUPS ...@@ -2,10 +2,10 @@ LPGROUPS
{ {
MODELNET_GRP MODELNET_GRP
{ {
repetitions="36"; repetitions="9";
server="2"; server="8";
modelnet_dragonfly="2"; modelnet_dragonfly="8";
dragonfly_router="1"; dragonfly_router="4";
} }
} }
PARAMS PARAMS
...@@ -25,5 +25,5 @@ PARAMS ...@@ -25,5 +25,5 @@ PARAMS
global_bandwidth="4.7"; global_bandwidth="4.7";
cn_bandwidth="5.25"; cn_bandwidth="5.25";
message_size="256"; message_size="256";
routing="adaptive"; routing="minimal";
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment