Commit ce2a8665 authored by Nikhil's avatar Nikhil Committed by Misbah Mubarak

Changes to Fat-tree network

-- Multi rail
-- Tapering
-- Within node handling
parent 933a1cf6
......@@ -91,6 +91,8 @@ void model_net_method_send_msg_recv_event_rc(tw_lp *sender);
// method - strange and disturbing things will happen otherwise
void model_net_method_idle_event(tw_stime offset_ts, int is_recv_queue,
tw_lp * lp);
void model_net_method_idle_event2(tw_stime offset_ts, int is_recv_queue,
int queue_offset, tw_lp * lp);
// Get a ptr to past the message struct area, where the self/remote events
// are located, given the type of network.
......@@ -115,6 +117,8 @@ typedef struct model_net_base_msg {
// no need for event type - in wrap message
model_net_request req;
int is_from_remote;
int isQueueReq;
tw_stime save_ts;
// parameters to pass to new messages (via model_net_set_msg_params)
// TODO: make this a union for multiple types of parameters
mn_sched_params sched_params;
......
......@@ -125,6 +125,7 @@ typedef struct model_net_request {
uint64_t msg_id;
int net_id;
int is_pull;
int queue_offset;
int remote_event_size;
int self_event_size;
char category[CATEGORY_NAME_MAX];
......
......@@ -49,6 +49,7 @@ struct fattree_message
// For buffer message
short vc_index;
short rail_id;
short vc_off;
int is_pull;
model_net_event_return event_rc;
......
......@@ -87,9 +87,15 @@ static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;
enum ROUTING_ALGO
{
STATIC=0,
STATIC=1,
ADAPTIVE,
};
enum RAIL_SELECTION_ALGO
{
RAIL_DEDICATED=1,
RAIL_ADAPTIVE,
};
static char routing_folder[MAX_NAME_LENGTH];
static char dot_file_p[MAX_NAME_LENGTH];
......@@ -141,14 +147,20 @@ struct fattree_param
int chunk_size; /* full-sized packets are broken into smaller chunks.*/
int packet_size;
int num_terminals;
int l1_set_size;
int l1_term_size;
int l1_set_size, l0_set_size;
int l1_term_size, l0_term_size;
int cn_per_switch;
double tapering_ratio;
double cn_delay;
double head_delay;
double credit_delay;
double router_delay;
double soft_delay;
int routing;
int rail_select;
int rail_size_limit;
int num_rails;
int ports_per_nic;
};
struct ftree_hash_key
......@@ -180,20 +192,20 @@ struct ft_terminal_state
// Fattree specific parameters
unsigned int terminal_id;
unsigned int switch_id;
tw_lpid switch_lp;
unsigned int rail_id;
tw_lpid *switch_lp;
// Each terminal will have an input and output channel with the switch
int vc_occupancy; // NUM_VC
tw_stime terminal_available_time;
tw_stime next_credit_available_time;
int *vc_occupancy; // NUM_VC
tw_stime* terminal_available_time;
struct mn_stats fattree_stats_array[CATEGORY_MAX];
fattree_message_list **terminal_msgs;
fattree_message_list **terminal_msgs_tail;
int terminal_length;
int in_send_loop;
int issueIdle;
int *terminal_length;
int *in_send_loop;
int *issueIdle;
struct rc_stack * st;
......@@ -210,8 +222,8 @@ struct ft_terminal_state
long finished_chunks;
long finished_packets;
tw_stime last_buf_full;
tw_stime busy_time;
tw_stime *last_buf_full;
tw_stime *busy_time;
char output_buf[4096];
char output_buf3[4096];
......@@ -246,6 +258,7 @@ struct switch_state
{
unsigned int switch_id;
int switch_level;
unsigned int rail_id;
int radix;
int num_cons;
int num_lcons;
......@@ -257,7 +270,6 @@ struct switch_state
int unused;
tw_stime* next_output_available_time;
tw_stime* next_credit_available_time;
tw_stime* last_buf_full;
tw_stime* busy_time;
......@@ -562,13 +574,14 @@ static void dot_write_switch_info(switch_state *s, int sw_gid, FILE *fout)
{
if(!s || s->unused || !(s->params) || !fout) return;
if(!dump_topo) return;
if(s->rail_id) return;
const char *root_attr = ",root_switch", *empty_attr = "";
uint64_t switch_guid = get_switch_guid(s);
fattree_param *p = s->params;
fprintf(fout, "\t\"S_%d_%d\" [comment=\"0x%016"PRIx64",radix=%d%s\"];\n",
s->switch_level, sw_gid, switch_guid, s->radix,
s->switch_level, s->switch_id, switch_guid, s->radix,
(p->num_levels == s->switch_level + 1) ? root_attr : empty_attr);
}
......@@ -577,6 +590,7 @@ static void dot_write_term_info(ft_terminal_state *t, FILE *fout)
{
uint64_t term_guid = t->terminal_id;
if(!dump_topo) return;
if(t->rail_id) return;
if(!t || !fout) return;
/* need to shift the term guid, because opensm doesn't like guid=0...0 */
......@@ -678,6 +692,7 @@ static void fattree_read_config(const char * anno, fattree_param *p){
p->ft_type = 1;
configuration_get_value_int(&config, "PARAMS", "ft_type", anno,
&p->ft_type);
if(!g_tw_mynode) printf("FT type is %d\n", p->ft_type);
configuration_get_value_int(&config, "PARAMS", "num_levels", anno,
&p->num_levels);
......@@ -687,6 +702,14 @@ static void fattree_read_config(const char * anno, fattree_param *p){
if(p->num_levels > 3) {
tw_error(TW_LOC, "Too many num_levels, only upto 3 supported Aborting\n");
}
if(!g_tw_mynode) printf("FT num levels is %d\n", p->num_levels);
p->tapering_ratio = 1;
configuration_get_value_double(&config, "PARAMS", "tapering", anno,
&p->tapering_ratio);
if(!g_tw_mynode) printf("FT tapering is %lf\n", p->tapering_ratio);
p->tapering_ratio += 1;
p->num_switches = (int *) malloc (p->num_levels * sizeof(int));
p->switch_radix = (int*) malloc (p->num_levels * sizeof(int));
......@@ -732,91 +755,107 @@ static void fattree_read_config(const char * anno, fattree_param *p){
token = strtok(NULL,",");
}
if(fmod(p->switch_radix[0], p->tapering_ratio)) {
tw_error(TW_LOC, "Tapering ratio does not fit the given radix; tapering + 1 should"
" evenly divide the L0 switch radix\n");
}
p->Ns = p->switch_radix[0]/2;
int num_terminals = p->num_switches[0]*p->switch_radix[0]/2;
if(p->num_switches[0] % p->Ns) {
if(!g_tw_mynode) printf("Number of switches has to be a multiple of switch radix/2 (%d);"
"next largest value is %d\n", p->Ns, (p->num_switches[0]/p->Ns + 1)*p->Ns);
tw_error(TW_LOC, "Incorrect number of switches");
}
int num_terminals = p->num_switches[0] * (p->tapering_ratio - 1)*(p->switch_radix[0]/p->tapering_ratio);
p->num_terminals = num_terminals;
p->l0_set_size = p->switch_radix[0]/2;
p->l0_term_size = (p->tapering_ratio - 1)*(p->switch_radix[0]/p->tapering_ratio);
if(p->num_levels == 2) {
p->num_switches[1] = p->num_switches[0]/2;
p->num_switches[1] = p->num_switches[0]/p->tapering_ratio/2;
p->switch_radix[1] = p->switch_radix[0];
p->l1_set_size = p->switch_radix[0]/4;
p->l1_term_size = (p->l1_set_size * p->switch_radix[0]);
p->l1_set_size = p->num_switches[1];
p->l1_term_size = num_terminals;
} else {
if(p->ft_type == 0){
p->Np = ceil((num_terminals)/(double)(p->Ns*p->Ns));
p->num_switches[1] = p->num_switches[0];
p->num_switches[2] = ceil((p->Ns*p->Ns) / floor(p->switch_radix[0]/p->Np));
p->switch_radix[1] = p->switch_radix[2] = p->switch_radix[0];
p->link_repetitions = floor(p->switch_radix[2] / p->Np);
printf("Np:%d Ns:%d\n",p->Np,p->Ns);
}else{
p->num_switches[1] = p->num_switches[0];
p->num_switches[2] = p->num_switches[0]/2;
p->switch_radix[1] = p->switch_radix[2] = p->switch_radix[0];
}
p->l1_set_size = p->switch_radix[0]/2;
p->l1_term_size = (p->l1_set_size * (p->switch_radix[0] / 2));
p->switch_radix[1] = p->switch_radix[2] = p->switch_radix[0];
p->l1_set_size = p->switch_radix[0]/p->tapering_ratio;
p->l1_term_size = p->l0_set_size * p->l0_term_size;
p->Np = p->num_switches[0]/p->l0_set_size;
p->num_switches[1] = p->Np * p->l1_set_size;
p->link_repetitions = p->switch_radix[0]/p->Np;
if(p->l1_set_size % p->link_repetitions) {
while(p->l1_set_size % p->link_repetitions) {
p->link_repetitions--;
}
}
int halfPods = p->l1_set_size/p->link_repetitions;
p->num_switches[2] = halfPods * p->Ns;
if(!g_tw_mynode) printf("Pods %d; L2 pods %d; Link repetition:%d; l1_set_size %d\n",
p->Np, halfPods, p->link_repetitions, p->l1_set_size);
}
if(p->ft_type == 1 && p->num_levels == 3) {
if(p->Np != p->switch_radix[0] && p->Np != p->switch_radix[0]/2) {
if(!g_tw_mynode) {
printf("ft_type 1 should be used for a system with %d or %d pods, which "
"needs %d or %d L0 switches, respectively\n", p->switch_radix[0],
p->switch_radix[0]/2, p->switch_radix[0] * p->switch_radix[0]/2,
p->switch_radix[0]/2 * p->switch_radix[0]/2);
}
tw_error(TW_LOC, "Incorrect number of switches for ft_type 1");
}
}
#if FATTREE_CONNECTIONS
for(int jj=0;jj<3;jj++)
for(int jj=0;jj<3 && !g_tw_mynode;jj++)
{
printf("num_switches[%d]=%d\n",jj,p->num_switches[jj]);
printf("switch_radix[%d]=%d\n",jj,p->switch_radix[jj]);
}
#endif
i = 1;
for(i = 1; i < p->num_levels - 1; i++) {
if(p->num_switches[i - 1] * p->switch_radix[i - 1] >
p->num_switches[i] * p->switch_radix[i]) {
tw_error(TW_LOC, "Not enough switches/radix at level %d for full "
"bisection bandwidth\n", i);
}
}
if(p->num_switches[i - 1] * p->switch_radix[i - 1] > 2 * p->num_switches[i] *
p->switch_radix[i]) {
tw_error(TW_LOC, "Not enough switches/radix at level %d (top) for full "
"bisection bandwidth\n", i);
}
configuration_get_value_int(&config, "PARAMS", "packet_size", anno,
rc = configuration_get_value_int(&config, "PARAMS", "packet_size", anno,
&p->packet_size);
if(!p->packet_size) {
if(rc) {
p->packet_size = 512;
fprintf(stderr, "Packet size is not specified, setting to %d\n",
p->packet_size);
}
if(!g_tw_mynode) printf("FT packet size is %d\n", p->packet_size);
p->num_rails = 1;
configuration_get_value_int(&config, "PARAMS", "num_rails", anno, &p->num_rails);
if(!g_tw_mynode) printf("FT num rails is %d\n", p->num_rails);
if(p->ft_type == 2) {
p->ports_per_nic = p->num_rails;
} else {
p->ports_per_nic = 1;
}
p->router_delay = 50;
configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
&p->router_delay);
if(!g_tw_mynode) printf("FT router delay is %.1lf\n", p->router_delay);
p->soft_delay = 1000;
configuration_get_value_double(&config, "PARAMS", "soft_delay", anno,
&p->soft_delay);
configuration_get_value_int(&config, "PARAMS", "vc_size", anno, &p->vc_size);
if(!p->vc_size) {
p->vc_size = 8*p->packet_size;
fprintf(stderr, "Buffer size of global channels not specified, setting to "
"%d\n", p->vc_size);
rc = configuration_get_value_int(&config, "PARAMS", "vc_size", anno, &p->vc_size);
if(rc) {
p->vc_size = 8 * p->packet_size;
}
if(!g_tw_mynode) printf("FT buffer size links is %d\n", p->vc_size);
configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno,
rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno,
&p->cn_vc_size);
if(!p->cn_vc_size) {
p->cn_vc_size = 8*p->packet_size;
fprintf(stderr, "Buffer size of compute node channels not specified, "
"setting to %d\n", p->cn_vc_size);
if(rc) {
p->cn_vc_size = 8 * p->packet_size;
}
if(!g_tw_mynode) printf("FT buffer size injection is %d\n", p->cn_vc_size);
rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
if(rc) {
p->chunk_size = 512;
fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
}
if(rc) {
p->chunk_size = 512;
}
char routing_str[MAX_NAME_LENGTH];
configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
......@@ -827,10 +866,26 @@ static void fattree_read_config(const char * anno, fattree_param *p){
p->routing = ADAPTIVE;
else
{
fprintf(stderr,
"No routing protocol specified, setting to adaptive routing\n");
p->routing = ADAPTIVE;
}
if(!g_tw_mynode) printf("FT routing is %d\n", p->routing);
char rail_select_str[MAX_NAME_LENGTH];
configuration_get_value(&config, "PARAMS", "rail_select", anno, rail_select_str,
MAX_NAME_LENGTH);
if(strcmp(rail_select_str, "dedicated") == 0)
p->rail_select = RAIL_DEDICATED;
else if(strcmp(rail_select_str, "adaptive")==0)
p->rail_select = RAIL_ADAPTIVE;
else {
p->rail_select = RAIL_DEDICATED;
}
if(!g_tw_mynode) printf("FT rail selection is %d\n", p->rail_select);
p->rail_size_limit = 32768;
configuration_get_value_int(&config, "PARAMS", "rail_select_limit", anno,
&p->rail_size_limit);
if(!g_tw_mynode) printf("FT rail select limit is %d\n", p->rail_size_limit);
configuration_get_value_int(&config, "PARAMS", "dump_topo", anno,
&dump_topo);
......@@ -855,21 +910,20 @@ static void fattree_read_config(const char * anno, fattree_param *p){
}
}
configuration_get_value_double(&config, "PARAMS", "link_bandwidth", anno,
rc = configuration_get_value_double(&config, "PARAMS", "link_bandwidth", anno,
&p->link_bandwidth);
if(!p->link_bandwidth) {
if(rc) {
p->link_bandwidth = 5;
fprintf(stderr, "Bandwidth of links is specified, setting to %lf\n",
p->link_bandwidth);
}
configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno,
rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno,
&p->cn_bandwidth);
if(!p->cn_bandwidth) {
p->cn_bandwidth = 5;
fprintf(stderr, "Bandwidth of compute node channels not specified, "
"setting to %lf\n", p->cn_bandwidth);
}
if(!g_tw_mynode) printf("FT bandwidths link %.1lf node %.1lf\n",
p->link_bandwidth, p->cn_bandwidth);
#if FATTREE_DEBUG
printf("l1_set_size:%d l1_term_size:%d\n",p->l1_set_size,p->l1_term_size);
......@@ -927,61 +981,83 @@ void ft_terminal_init( ft_terminal_state * s, tw_lp * lp )
int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM,
s->anno, 0);
if(num_lps != (s->params->switch_radix[0]/2)) {
tw_error(TW_LOC, "Number of NICs per repetition has to be equal to "
"half the radix of leaf level switches %d vs %d\n", num_lps,
s->params->switch_radix[0]/2);
if(s->params->ft_type != 2) {
num_lps /= s->params->num_rails;
}
s->terminal_id = (mapping_rep_id * num_lps) + mapping_offset;
s->switch_id = s->terminal_id / (s->params->switch_radix[0] / 2);
codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
s->switch_id, 0, &s->switch_lp);
s->terminal_available_time = 0.0;
s->packet_counter = 0;
if(num_lps != s->params->l0_term_size) {
tw_error(TW_LOC, "Number of NICs per repetition per rail has to be equal to "
"%d, not the given value of %d\n", s->params->l0_term_size, num_lps);
}
if(s->params->ft_type != 2) {
s->terminal_id = (mapping_rep_id * num_lps) + (mapping_offset/s->params->num_rails);
s->rail_id = (mapping_offset % s->params->num_rails);
} else {
s->terminal_id = (mapping_rep_id * num_lps) + mapping_offset;
s->rail_id = 0;
}
s->switch_id = s->terminal_id / s->params->l0_term_size;
s->switch_lp = (tw_lpid*)malloc(s->params->ports_per_nic * sizeof(tw_lpid));
if(s->params->ft_type != 2) {
codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
s->switch_id, 0 + s->params->num_levels * s->rail_id, &s->switch_lp[0]);
} else {
for(int i = 0; i < s->params->ports_per_nic; i++) {
codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
s->switch_id, 0 + s->params->num_levels * i, &s->switch_lp[i]);
}
}
s->terminal_available_time = (tw_stime*) malloc(s->params->ports_per_nic * sizeof(tw_stime));
s->vc_occupancy = (int*) malloc(s->params->ports_per_nic * sizeof(int));
s->terminal_length = (int*) malloc(s->params->ports_per_nic * sizeof(int));
s->in_send_loop = (int*) malloc(s->params->ports_per_nic * sizeof(int));
s->issueIdle = (int*) malloc(s->params->ports_per_nic * sizeof(int));
s->terminal_msgs =
(fattree_message_list**)malloc(1*sizeof(fattree_message_list*));
(fattree_message_list**)malloc(s->params->ports_per_nic*sizeof(fattree_message_list*));
s->terminal_msgs_tail =
(fattree_message_list**)malloc(1*sizeof(fattree_message_list*));
(fattree_message_list**)malloc(s->params->ports_per_nic*sizeof(fattree_message_list*));
s->last_buf_full = (tw_stime*) malloc(s->params->ports_per_nic * sizeof(tw_stime));
s->busy_time = (tw_stime*) malloc(s->params->ports_per_nic * sizeof(tw_stime));
for(int i = 0; i < s->params->ports_per_nic; i++) {
s->terminal_available_time[i] = 0.0;
s->vc_occupancy[i] = 0;
s->terminal_msgs[i] = NULL;
s->terminal_msgs_tail[i] = NULL;
s->terminal_length[i] = 0;
s->in_send_loop[i] = 0;
s->issueIdle[i] = 0;
s->last_buf_full[i] = 0;
s->busy_time[i] = 0;
}
s->packet_counter = 0;
s->finished_msgs = 0;
s->finished_chunks = 0;
s->finished_packets = 0;
s->total_time = 0.0;
s->total_msg_size = 0;
s->last_buf_full = 0;
s->busy_time = 0;
#if FATTREE_HELLO
printf("I am terminal %d (%llu), connected to switch %d\n", s->terminal_id,
LLU(lp->gid), s->switch_id);
#if FATTREE_DEBUG
printf("I am terminal %d (%llu), connected to switch %d in rail id %d\n", s->terminal_id,
LLU(lp->gid), s->switch_id, s->rail_id);
#endif
rc_stack_create(&s->st);
s->vc_occupancy = 0;
s->terminal_msgs[0] = NULL;
s->terminal_msgs_tail[0] = NULL;
s->terminal_length = 0;
s->in_send_loop = 0;
s->issueIdle = 0;
s->rank_tbl = NULL;
//if(!s->rank_tbl)
// tw_error(TW_LOC, "\n Hash table not initialized! ");
s->params->num_terminals = codes_mapping_get_lp_count(lp_group_name, 0,
LP_CONFIG_NM, s->anno, 0);
/* dump partial topology into DOT format
* skip term2sw link part, because we are missing the remote switch port
* in this stage of the code
*/
if(!dot_file)
if(!dot_file && !s->rail_id)
dot_write_open_file(&dot_file);
dot_write_term_info(s, dot_file);
/* ensure the DOT file is out before reading it with external tools */
if(dot_file)
if(dot_file && !s->rail_id)
fflush(dot_file);
return;
......@@ -992,14 +1068,11 @@ void switch_init(switch_state * r, tw_lp * lp)
{
char anno[MAX_NAME_LENGTH];
int num_terminals = -1;
if(def_gname_set == 0) {
def_gname_set = 1;
codes_mapping_get_lp_info(0, def_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
num_terminals = codes_mapping_get_lp_count(def_group_name, 0,
LP_CONFIG_NM, anno, 0);
}
codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
......@@ -1016,29 +1089,28 @@ void switch_init(switch_state * r, tw_lp * lp)
// shorthand
fattree_param *p = r->params;
if(mapping_offset == p->num_levels - 1) {
if(mapping_rep_id >= p->num_switches[mapping_offset]) {
r->unused = 1;
return;
}
if(mapping_rep_id >= p->num_switches[mapping_offset % p->num_levels]) {
r->unused = 1;
return;
}
r->unused = 0;
r->switch_id = mapping_rep_id + mapping_offset * p->num_switches[0];
if(num_terminals != -1) {
p->num_terminals = num_terminals;
if((mapping_offset % p->num_levels) == 0) {
r->switch_id = mapping_rep_id;
} else if((mapping_offset % p->num_levels) == 1) {
r->switch_id = mapping_rep_id + p->num_switches[0];
} else {
r->switch_id = mapping_rep_id + p->num_switches[0] + p->num_switches[1];
}
r->rail_id = mapping_offset / p->num_levels;
r->switch_level = mapping_offset;
r->switch_level = mapping_offset % p->num_levels;
r->radix = p->switch_radix[r->switch_level];
r->next_output_available_time = (tw_stime*) malloc (r->radix *
sizeof(tw_stime));
r->next_credit_available_time = (tw_stime*) malloc (r->radix *
sizeof(tw_stime));
r->vc_occupancy = (int*) malloc (r->radix * sizeof(int));
r->in_send_loop = (int*) malloc (r->radix * sizeof(int));
r->link_traffic = (int64_t*) malloc (r->radix * sizeof(int64_t));
......@@ -1067,7 +1139,6 @@ void switch_init(switch_state * r, tw_lp * lp)
r->busy_time[i] = 0.0;
r->busy_time_sample[i] = 0.0;
r->next_output_available_time[i] = 0;
r->next_credit_available_time[i] = 0;
r->vc_occupancy[i] = 0;
r->in_send_loop[i] = 0;
r->link_traffic[i] = 0;
......@@ -1079,7 +1150,7 @@ void switch_init(switch_state * r, tw_lp * lp)
}
/* dump partial topology info into DOT format (switch radix, guid, ...) */
if(!dot_file)
if(!dot_file && !r->rail_id)
dot_write_open_file(&dot_file);
dot_write_switch_info(r, lp->gid, dot_file);
......@@ -1092,228 +1163,185 @@ void switch_init(switch_state * r, tw_lp * lp)
//set lps connected to each port
r->num_cons = 0;
r->num_lcons = 0;
#if FATTREE_HELLO
printf("I am switch %d (%llu), level %d, radix %d\n", r->switch_id,
LLU(lp->gid), r->switch_level, r->radix);
#if FATTREE_DEBUG
printf("I am switch %d (%llu), level %d, radix %d, rail id %d\n", r->switch_id,
LLU(lp->gid), r->switch_level, r->radix, r->rail_id);
#endif
//if at level 0, first half ports go to terminals
if(r->switch_level == 0) {
int start_terminal = r->switch_id * (p->switch_radix[0] / 2);
int end_terminal = start_terminal + (p->switch_radix[0] / 2);
int term_rails, term_railid;
if(p->ft_type != 2) {
term_rails = p->num_rails;
term_railid = r->rail_id;
} else {
term_rails = 1;
term_railid = 0;
}
int start_terminal = r->switch_id * p->l0_term_size;
int end_terminal = start_terminal + p->l0_term_size;
for(int term = start_terminal; term < end_terminal; term++) {
tw_lpid nextTerm;
int rep = term / (p->switch_radix[0] / 2);
int off = term % (p->switch_radix[0] / 2);
int rep = term / p->l0_term_size;
int off = (term % p->l0_term_size) * term_rails + term_railid;
codes_mapping_get_lp_id(def_group_name, LP_CONFIG_NM, NULL, 1,
rep, off, &nextTerm);
r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_CONNECTIONS
written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(codes_mapping_get_lp_relative_id(nextTerm,0,0)));
written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(codes_mapping_get_lp_relative_id(nextTerm,0,0)/term_rails));
#endif
r->num_lcons++;
#if FATTREE_DEBUG
printf("L0->term I am switch %d, connect to terminal %d (%llu) at port %d yes collecting\n",
r->switch_id, term, LLU(nextTerm), r->num_cons - 1);
printf("L0->term I am switch %d, connect to terminal %d (%llu) at port %d, rail id %d yes collecting\n",
r->switch_id, term, LLU(nextTerm), r->num_cons - 1, r->rail_id);
#endif
/* write sw2term links and (reverse link, too) into DOT file */
dot_write_sw2term_link(r->switch_level, lp->gid, r->num_cons - 1, term, 1, dot_file);
if(!r->rail_id) {
dot_write_sw2term_link(r->switch_level, r->switch_id, r->num_cons - 1, term, 1, dot_file);
}
}
r->start_lneigh = start_terminal;
r->end_lneigh = end_terminal;
r->con_per_lneigh = 1;
assert(r->num_lcons == (r->radix / 2));
assert(r->num_lcons == p->l0_term_size);
int l1_set;
if(p->num_levels == 2) {
l1_set = 0;
r->con_per_uneigh = 2;
} else {
l1_set = r->switch_id / p->l1_set_size;
l1_set = r->switch_id / p->l0_set_size;
r->con_per_uneigh = 1;
}
int l1_base = l1_set * p->l1_set_size;
r->start_uneigh = p->num_switches[0] + l1_base;
for(int l1 = 0; l1 < p->l1_set_size; l1++) {
tw_lpid nextTerm;
assert(l1_base < p->num_switches[1]);
codes_mapping_get_lp_id(lp_group_name, "fattree_switch", NULL, 1,
l1_base, 1, &nextTerm);
l1_base, 1 + r->rail_id * p->num_levels, &nextTerm);
for(int con = 0; con < r->con_per_uneigh; con++) {
r->port_connections[r->num_cons++] = nextTerm;
#if FATTREE_CONNECTIONS || FATTREE_DEBUG
#if FATTREE_CONNECTIONS
codes_mapping_get_lp_info(nextTerm, lp_group_name, &mapping_grp_id, NULL,
&mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
next_switch_lid = mapping_rep_id + mapping_offset * p->num_switches[0];
written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(next_switch_lid)+p->num_terminals);
next_switch_lid = mapping_rep_id + p->num_switches[0];
written += sprintf(r->output_buf + written, "%u, %llu, ", r->switch_id+p->num_terminals,LLU(next_switch_lid)+p->num_terminals);
#endif
#if FATTREE_DEBUG
printf("L0->L1 I am switch %d, connect to upper switch %d L1 (%llu) rel_id:%llu at port %d yes collecting\n",
r->switch_id, l1_base, LLU(nextTerm), LLU(next_switch_lid), r->num_cons - 1);
printf("L0->L1 I am switch %d, connect to upper switch %d L1 (%llu) rel_id:%llu at port %d, rail id %d yes collecting\n",
r->switch_id, l1_base, LLU(nextTerm), LLU(next_switch_lid), r->num_cons - 1, r->rail_id);
#endif
/* write all inter switch links to DOT file (initialized before) */
dot_write_sw2sw_link(r->switch_level, lp->gid, r->num_cons - 1,
r->switch_level + 1, nextTerm, -1, dot_file);
if(!r->rail_id) {
dot_write_sw2sw_link(r->switch_level, r->switch_id, r->num_cons - 1,
r->switch_level + 1, p->num_switches[0] + l1_base, -1, dot_file);
}
}
l1_base++;
}
} else if (r->switch_level == 1) {
int l0_set_size, l0_base;
int l0_base;