Commit f24bd5c2 authored by Neil McGlohon's avatar Neil McGlohon

Bring in updated slimfly model from fitfly fork

This commit brings in changes that were made to the slimfly
model by Noah since his fork to add support of the fitfly
variant. This introduces dual-rail/dual-plane behavior. It
should not interfere with single-rail/plane behavior of
the model if configured that way.

Almost all changes were incremental with only a few changes from
mainline that were brought in - specifically vis api and damaris
support changes.
parent 28cc07b1
......@@ -63,6 +63,7 @@ struct slim_terminal_message
// For buffer message
short vc_index;
short rail_id;
int sender_radix;
int output_chan;
model_net_event_return event_rc;
......
......@@ -28,13 +28,15 @@
#define SLIMFLY_HASH_TABLE_SIZE 65536
// debugging parameters
#define TRACK 4
#define TRACK -9
//#define TRACK 100001
#define TRACK_MSG 0
#define TRACK_OUTPUT 1
#define TRACK_OUTPUT 0
#define DEBUG 0
#define DEBUG_ROUTING 0
#define LOAD_FROM_FILE 0
#define SLIMFLY_CONNECTIONS 1
#define MSG_TIMES 0 //Collects msg send times and outputs lp-io-dir
#define LP_CONFIG_NM (model_net_lp_config_names[SLIMFLY])
#define LP_METHOD_NM (model_net_method_names[SLIMFLY])
......@@ -44,7 +46,6 @@
#define ROUTER_SENDS_RECVS_LOG 0
#define TERMINAL_OCCUPANCY_LOG 0
#define ROUTER_OCCUPANCY_LOG 0
#define PARAMS_LOG 0
#define N_COLLECT_POINTS 100
/*unsigned long terminal_sends[TEMP_NUM_TERMINALS][N_COLLECT_POINTS];
......@@ -70,8 +71,6 @@ FILE * MMS_input_file=NULL;
int csf_ratio = 1; //Constant selected to balance the ratio between minimal and indirect routes
int num_indirect_routes = 4; //Number of indirect (Valiant) routes to use in Adaptive routing methods
float adaptive_threshold = 0.1;
float pe_throughput_percent = 0.0;
float pe_throughput = 0.0;
int *X;
int *X_prime;
......@@ -83,6 +82,8 @@ static double maxd(double a, double b) { return a < b ? b : a; }
/* minimal and non-minimal packet counts for adaptive routing*/
static int minimal_count=0, nonmin_count=0;
long slimfly_packet_gen = 0, slimfly_packet_fin = 0;
typedef struct slimfly_param slimfly_param;
/* annotation-specific parameters (unannotated entry occurs at the
* last index) */
......@@ -123,6 +124,10 @@ void slim_delete_terminal_message_list(slim_terminal_message_list *this) {
struct slimfly_param
{
int sf_type;
int num_rails;
int ports_per_nic;
int rail_select; /*Selection of rail routing method*/
// configuration parameters
int num_routers; /*NUM_ROUTER Number of routers in a group*/
double local_bandwidth; /*LOCAL_BANDWIDTH bandwidth of the router-router channels within a group */
......@@ -175,24 +180,34 @@ struct terminal_state
{
uint64_t packet_counter;
int packet_gen;
int packet_fin;
#if MSG_TIMES
int * msg_send_times;
int * msg_rail_select;
char output_buf2[4096];
#endif
int router_id;
tw_lpid * router_lp;
int terminal_id;
// Each terminal will have an input and output channel with the router
int* vc_occupancy; // NUM_VC
int num_vcs;
tw_stime terminal_available_time;
tw_stime* terminal_available_time;
slim_terminal_message_list **terminal_msgs;
slim_terminal_message_list **terminal_msgs_tail;
int in_send_loop;
int *in_send_loop;
// Terminal generate, sends and arrival T_SEND, T_ARRIVAL, T_GENERATE
// Router-Router Intra-group sends and receives RR_LSEND, RR_LARRIVE
// Router-Router Inter-group sends and receives RR_GSEND, RR_GARRIVE
struct mn_stats slimfly_stats_array[CATEGORY_MAX];
struct rc_stack * st;
int issueIdle;
int terminal_length;
int *issueIdle;
int *terminal_length;
const char * anno;
const slimfly_param *params;
......@@ -201,14 +216,14 @@ struct terminal_state
uint64_t rank_tbl_pop;
tw_stime total_time;
long total_msg_size;
long total_hops;
uint64_t total_msg_size;
double total_hops;
long finished_msgs;
long finished_chunks;
long finished_packets;
tw_stime * last_buf_full;
tw_stime busy_time;
tw_stime *last_buf_full;
tw_stime *busy_time;
char output_buf[512];
};
......@@ -251,6 +266,13 @@ enum ROUTING_ALGO
ADAPTIVE
};
enum RAIL_SELECTION_ALGO
{
RAIL_CONGESTION=1, // Selects rail with minimal injection congestion
RAIL_PATH, // Selects the rail that provides minimal path congestion is tie breaker
RAIL_DEDICATED // Selects a specific rail
};
struct router_state
{
int router_id;
......@@ -305,6 +327,9 @@ static void ross_slimfly_sample_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, s
static void ross_slimfly_sample_rc_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_cn_sample *sample);
static void ross_slimfly_rsample_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_router_sample *sample);
static void ross_slimfly_rsample_rc_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_router_sample *sample);
int get_path_length_from_terminal(int src, int dest, const slimfly_param *p);
void get_router_connections(int src_router_id, int num_global_channels, int num_local_channels,
int total_routers, int* local_channels, int* global_channels, int sf_type);
st_model_types slimfly_model_types[] = {
{(ev_trace_f) slimfly_event_collect,
......@@ -461,6 +486,18 @@ static void slimfly_read_config(const char * anno, slimfly_param *params){
// shorthand
slimfly_param *p = params;
p->sf_type = 1;
configuration_get_value_int(&config, "PARAMS", "sf_type", anno, &p->sf_type);
if(!g_tw_mynode) printf("SF type is %d\n", p->sf_type);
p->num_rails = 1;
configuration_get_value_int(&config, "PARAMS", "num_rails", anno, &p->num_rails);
if(!g_tw_mynode) printf("FT num rails is %d\n", p->num_rails);
p->ports_per_nic = 1;
if(p->num_rails > 1)
p->ports_per_nic = 2;
configuration_get_value_int(&config, "PARAMS", "num_routers", anno,
&p->num_routers);
if(p->num_routers <= 0) {
......@@ -588,6 +625,20 @@ static void slimfly_read_config(const char * anno, slimfly_param *params){
routing = -1;
}
char rail_select_str[MAX_NAME_LENGTH];
configuration_get_value(&config, "PARAMS", "rail_select", anno, rail_select_str,
MAX_NAME_LENGTH);
if(strcmp(rail_select_str, "dedicated") == 0)
p->rail_select = RAIL_DEDICATED;
else if(strcmp(rail_select_str, "congestion")==0)
p->rail_select = RAIL_CONGESTION;
else if(strcmp(rail_select_str, "path")==0)
p->rail_select = RAIL_PATH;
else {
p->rail_select = RAIL_DEDICATED;
}
if(!g_tw_mynode) printf("SF rail selection is %d\n", p->rail_select);
TEMP_RADIX = p->num_local_channels + p->num_global_channels + p->num_cn;
TEMP_NUM_VC = p->num_vcs;
......@@ -635,8 +686,7 @@ static void slimfly_report_stats()
long long total_finished_msgs, final_msg_sz;
tw_stime avg_time, max_time;
int total_minimal_packets, total_nonmin_packets;
float throughput_avg = 0.0;
float throughput_avg2 = 0.0;
long total_gen, total_fin;
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
......@@ -646,8 +696,8 @@ static void slimfly_report_stats()
MPI_Reduce( &slimfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &slimfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&pe_throughput_percent, &throughput_avg, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&pe_throughput, &throughput_avg2, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &slimfly_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &slimfly_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
......@@ -655,21 +705,10 @@ static void slimfly_report_stats()
/* print statistics */
if(!g_tw_mynode)
{
printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld \n", (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs);
#if PARAMS_LOG
throughput_avg = throughput_avg / (float)slim_total_terminals_noah;
throughput_avg2 = throughput_avg2 / (float)slim_total_terminals_noah;
//Open file to append simulation results
sprintf( log, "slimfly-results-log.txt");
slimfly_results_log=fopen(log, "a");
if(slimfly_results_log == NULL)
tw_error(TW_LOC, "\n Failed to open slimfly results log file \n");
printf("Printing Simulation Parameters/Results Log File\n");
fprintf(slimfly_results_log,"%10.3lf, %15.3lf, %11.3lf, %13.3d, %16.3d, %16.3lld, %25.5f, %14.5f, ", (float)avg_hops/total_finished_packets, avg_time/(total_finished_packets),max_time,total_minimal_packets,total_nonmin_packets,total_finished_chunks,throughput_avg*100,throughput_avg2);
fclose(slimfly_results_log);
#endif
printf(" Total packets routed minimally: %d. Total packets routed non-minimally: %d\n",total_minimal_packets, total_nonmin_packets);
printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld finished chunks %lld \n",
(float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs, total_finished_chunks);
printf(" Total packets generated %ld finished %ld \n", total_gen, total_fin);
}
#if ROUTER_OCCUPANCY_LOG
......@@ -793,8 +832,9 @@ static void slimfly_report_stats()
void slim_terminal_init( terminal_state * s,
tw_lp * lp )
{
s->packet_gen = 0;
s->packet_fin = 0;
int i;
char anno[MAX_NAME_LENGTH];
// Assign the global router ID
......@@ -814,45 +854,76 @@ void slim_terminal_init( terminal_state * s,
int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM,
s->anno, 0);
int num_routers = codes_mapping_get_lp_count(lp_group_name, 0 ,"slimfly_router",
s->anno, 0);
#if MSG_TIMES
s->msg_send_times = (int*)calloc(200,sizeof(int));
s->msg_rail_select = (int*)calloc(200,sizeof(int));
#endif
s->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
s->router_id=(int)s->terminal_id / (num_lps);
s->terminal_available_time = 0.0;
s->packet_counter = 0;
s->router_lp=(tw_lpid*)malloc(s->params->ports_per_nic * sizeof(tw_lpid));
//Assign router from first rail
codes_mapping_get_lp_id(lp_group_name, "slimfly_router", NULL, 1,
s->router_id, 0, &s->router_lp[0]);
//s->router_lp[0] = codes_mapping_get_lpid_from_relative(s->router_id,
// lp_group_name, "slimfly_router", NULL, 0);
#if SLIMFLY_CONNECTIONS
int written = 0;
written += sprintf(s->output_buf + written, "%d, %d, ", s->terminal_id, s->params->slim_total_terminals + s->router_id);
#endif
//Assign router from second rail
if(s->params->sf_type == 1){
codes_mapping_get_lp_id(lp_group_name, "slimfly_router", NULL, 1,
(num_routers-1) - s->router_id - (num_routers/2), 1, &s->router_lp[1]);
#if SLIMFLY_CONNECTIONS
written += sprintf(s->output_buf + written, "%d, %d, ", s->terminal_id, s->params->slim_total_terminals + (num_routers-1) - s->router_id);
#endif
}
#if SLIMFLY_CONNECTIONS
lp_io_write(lp->gid, "slimfly-config-terminal-connections", written, s->output_buf);
#endif
if(s->terminal_id == 1723)
printf("terminal_id:%d, router_id:%d, router_lp[0]:%d, router_lp[1]:%d\n",s->terminal_id, s->router_id, (int)s->router_lp[0], (int)s->router_lp[1]);
s->terminal_available_time = (tw_stime*)malloc(s->params->ports_per_nic * sizeof(tw_stime));
s->busy_time = (tw_stime*)malloc(s->params->ports_per_nic * sizeof(tw_stime));
s->issueIdle = (int*)malloc(s->params->ports_per_nic * sizeof(int));
s->in_send_loop = (int*)malloc(s->params->ports_per_nic * sizeof(int));
s->terminal_length = (int*)malloc(s->params->ports_per_nic * sizeof(int));
s->vc_occupancy = (int*)malloc(s->params->ports_per_nic * sizeof(int*));
s->last_buf_full = (tw_stime*)malloc(s->params->ports_per_nic * sizeof(tw_stime*));
s->terminal_msgs = (slim_terminal_message_list**)malloc(s->params->ports_per_nic * sizeof(slim_terminal_message_list*));
s->terminal_msgs_tail = (slim_terminal_message_list**)malloc(s->params->ports_per_nic * sizeof(slim_terminal_message_list*));
for(int i = 0; i < s->params->ports_per_nic; i++) {
s->terminal_available_time[i] = 0.0;
s->vc_occupancy[i] = 0;
s->last_buf_full[i] = 0;
s->terminal_msgs[i] = NULL;
s->terminal_msgs_tail[i] = NULL;
s->terminal_length[i] = 0;
s->in_send_loop[i] = 0;
s->issueIdle[i] = 0;
s->busy_time[i] = 0;
}
s->packet_counter = 0;
s->finished_msgs = 0;
s->finished_chunks = 0;
s->finished_packets = 0;
s->total_time = 0.0;
s->total_msg_size = 0;
s->busy_time = 0;
rc_stack_create(&s->st);
s->num_vcs = 1;
s->last_buf_full = (tw_stime*)malloc(s->num_vcs * sizeof(tw_stime));
s->vc_occupancy = (int*)malloc(s->num_vcs * sizeof(int));
for( i = 0; i < s->num_vcs; i++ )
{
s->last_buf_full[i] = 0.0;
s->vc_occupancy[i]=0;
}
s->rank_tbl = qhash_init(slimfly_rank_hash_compare, slimfly_hash_func, SLIMFLY_HASH_TABLE_SIZE);
if(!s->rank_tbl)
tw_error(TW_LOC, "\n Hash table not initialized! ");
s->terminal_msgs =
(slim_terminal_message_list**)malloc(1*sizeof(slim_terminal_message_list*));
s->terminal_msgs_tail =
(slim_terminal_message_list**)malloc(1*sizeof(slim_terminal_message_list*));
s->terminal_msgs[0] = NULL;
s->terminal_msgs_tail[0] = NULL;
s->terminal_length = 0;
s->in_send_loop = 0;
s->issueIdle = 0;
return;
}
......@@ -877,7 +948,7 @@ void slim_router_setup(router_state * r, tw_lp * lp)
// shorthand
const slimfly_param *p = r->params;
r->router_id=mapping_rep_id + mapping_offset;
r->router_id=mapping_rep_id + mapping_offset * r->params->slim_total_routers;
r->group_id=r->router_id/p->num_routers;
r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int));
......@@ -902,8 +973,8 @@ void slim_router_setup(router_state * r, tw_lp * lp)
r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
// ROSS Instrumentation
if (g_st_use_analysis_lps && g_st_model_stats)
lp->model_types->sample_struct_sz = sizeof(struct slimfly_router_sample) + sizeof(int) * p->radix;
if (g_st_use_analysis_lps && g_st_model_stats)
lp->model_types->sample_struct_sz = sizeof(struct slimfly_router_sample) + sizeof(int) * p->radix;
rc_stack_create(&r->st);
......@@ -978,124 +1049,23 @@ void slim_router_setup(router_state * r, tw_lp * lp)
r->global_channel[i] = one_number;
}
fclose(MMS_input_file);
#else
//Compute MMS router layout/connection graph
int rid_s = (int)r->router_id; // ID for source router
int rid_d; // ID for dest. router
int s_s,s_d; // subgraph location for source and destination routers
int i_s,i_d; // x or m coordinates for source and destination routers
int j_s,j_d; // y or c coordinates for source and destination routers
int k;
int local_idx = 0;
int global_idx = 0;
int generator_size = X_size;
get_router_connections(r->router_id, p->num_global_channels, p->num_local_channels,
p->slim_total_routers, r->local_channel, r->global_channel, p->sf_type);
#endif
for(rid_d=0;rid_d<r->params->slim_total_routers;rid_d++)
{
// Decompose source and destination Router IDs into 3D subgraph coordinates (subgraph,i,j)
if(rid_d >= r->params->slim_total_routers/2)
{
s_d = 1;
i_d = (rid_d - r->params->slim_total_routers/2) / r->params->num_global_channels;
j_d = (rid_d - r->params->slim_total_routers/2) % r->params->num_global_channels;
}
else
{
s_d = 0;
i_d = rid_d / r->params->num_global_channels;
j_d = rid_d % r->params->num_global_channels;
}
if(rid_s >= r->params->slim_total_routers/2)
{
s_s = 1;
i_s = (rid_s - r->params->slim_total_routers/2) / r->params->num_global_channels;
j_s = (rid_s - r->params->slim_total_routers/2) % r->params->num_global_channels;
}
else
{
s_s = 0;
i_s = rid_s / r->params->num_global_channels;
j_s = rid_s % r->params->num_global_channels;
}
// Check for subgraph 0 local connections
if(s_s==0 && s_d==0)
{
if(i_s==i_d) // equation (2) y-y' is in X'
{
for(k=0;k<generator_size;k++)
{
if(abs(j_s-j_d)==X[k])
{
r->local_channel[local_idx++] = rid_d;
}
}
}
}
// Check if global connections
if(s_s==0 && s_d==1)
{
if(j_s == (i_d*i_s + j_d) % r->params->num_routers) // equation (3) y=mx+c
{
r->global_channel[global_idx++] = rid_d;
}
}
}
#if SLIMFLY_CONNECTIONS
int written = 0;
int i;
// Loop over second subgraph source routers
for(rid_d=r->params->slim_total_routers-1;rid_d>=0;rid_d--)
{
// Decompose source and destination Router IDs into 3D subgraph coordinates (subgraph,i,j)
if(rid_d >= r->params->slim_total_routers/2)
{
s_d = 1;
i_d = (rid_d - r->params->slim_total_routers/2) / r->params->num_global_channels;
j_d = (rid_d - r->params->slim_total_routers/2) % r->params->num_global_channels;
}
else
{
s_d = 0;
i_d = rid_d / r->params->num_global_channels;
j_d = rid_d % r->params->num_global_channels;
}
if(rid_s >= r->params->slim_total_routers/2)
{
s_s = 1;
i_s = (rid_s - r->params->slim_total_routers/2) / r->params->num_global_channels;
j_s = (rid_s - r->params->slim_total_routers/2) % r->params->num_global_channels;
}
else
{
s_s = 0;
i_s = rid_s / r->params->num_global_channels;
j_s = rid_s % r->params->num_global_channels;
}
// Check for subgraph 1 local connections
if(s_s==1 && s_d==1)
{
if(i_s==i_d) // equation (2) c-c' is in X'
{
for(k=0;k<generator_size;k++)
{
if(abs(j_s-j_d)==X_prime[k])
{
r->local_channel[local_idx++] = rid_d;
}
}
}
}
// Check if global connections
if(s_s==1 && s_d==0)
{
if(j_d == (i_s*i_d + j_s) % r->params->num_routers) // equation (3) y=mx+c
{
r->global_channel[global_idx++] = rid_d;
}
}
for(i=0;i<r->params->num_local_channels;i++){
written += sprintf(r->output_buf + written, "%d, %d, ", r->params->slim_total_terminals + r->router_id, r->params->slim_total_terminals + r->local_channel[i]);
}
for(i=0;i<r->params->num_global_channels;i++){
written += sprintf(r->output_buf + written, "%d, %d, ", r->params->slim_total_terminals + r->router_id, r->params->slim_total_terminals + r->global_channel[i]);
}
lp_io_write(lp->gid, "slimfly-config-router-connections", written, r->output_buf);
#endif
assert(local_idx == r->params->num_local_channels);
assert(global_idx == r->params->num_global_channels);
return;
}
......@@ -1120,10 +1090,13 @@ static tw_stime slimfly_packet_event(
tw_stime xfer_to_nic_time;
slim_terminal_message * msg;
char* tmp_ptr;
#if DEBUG
printf("\x1B[34m(%lf) packet_event() msg_id:%lu rail_id:%d\x1b[0m\n",tw_now(sender), req->msg_id, req->queue_offset);
#endif
xfer_to_nic_time = codes_local_latency(sender);
//e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender);
//msg = tw_event_data(e_new);
//printf("%llu packet_event() xfer to nic time: %llu, offset: %llu\n",LLU(tw_now(sender)),LLU(xfer_to_nic_time),LLU(offset));
e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
sender, SLIMFLY, (void**)&msg, (void**)&tmp_ptr);
strcpy(msg->category, req->category);
......@@ -1137,6 +1110,7 @@ static tw_stime slimfly_packet_event(
msg->local_event_size_bytes = 0;
msg->type = T_GENERATE;
msg->dest_terminal_id = req->dest_mn_lp;
msg->rail_id = req->queue_offset;
msg->message_id = req->msg_id;
msg->is_pull = req->is_pull;
msg->pull_size = req->pull_size;
......@@ -1229,6 +1203,9 @@ void slim_router_credit_send(router_state * s, slim_terminal_message * msg, tw_l
void slim_packet_generate_rc(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
s->packet_gen--;
slimfly_packet_gen--;
tw_rand_reverse_unif(lp->rng);
int num_chunks = msg->packet_size/s->params->chunk_size;
......@@ -1243,19 +1220,19 @@ void slim_packet_generate_rc(terminal_state * s, tw_bf * bf, slim_terminal_messa
{
slim_delete_terminal_message_list(return_tail(s->terminal_msgs,
s->terminal_msgs_tail, 0));
s->terminal_length -= s->params->chunk_size;
s->terminal_length[msg->saved_vc] -= s->params->chunk_size;
}
if(bf->c5)
{
codes_local_latency_reverse(lp);
s->in_send_loop = 0;
s->in_send_loop[msg->saved_vc] = 0;
}
if(bf->c11)
{
s->issueIdle = 0;
s->issueIdle[msg->rail_id] = 0;
if(bf->c8)
{
s->last_buf_full[0] = msg->saved_busy_time;
s->last_buf_full[msg->rail_id] = msg->saved_busy_time;
}
}
struct mn_stats* stat;
......@@ -1268,6 +1245,11 @@ void slim_packet_generate_rc(terminal_state * s, tw_bf * bf, slim_terminal_messa
/* generates packet at the current slimfly compute node */
void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
slimfly_packet_gen++;
s->packet_gen++;
//printf("%llu packet_generate()\n",LLU(tw_now(lp)));
tw_stime ts, nic_ts;
assert(lp->gid != msg->dest_terminal_id);
......@@ -1276,13 +1258,15 @@ void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message
uint64_t i;
int total_event_size;
uint64_t num_chunks = msg->packet_size / p->chunk_size;
if (msg->packet_size % s->params->chunk_size)
double cn_delay = s->params->cn_delay;
if (msg->packet_size < (unsigned)s->params->chunk_size)
num_chunks++;
if(!num_chunks)
num_chunks = 1;
if (msg->packet_size < (unsigned)s->params->chunk_size)
cn_delay = bytes_to_ns(msg->packet_size % s->params->chunk_size, s->params->cn_bandwidth);
nic_ts = g_tw_lookahead + (s->params->cn_delay * num_chunks) + tw_rand_unif(lp->rng);
nic_ts = g_tw_lookahead + (num_chunks * cn_delay) + tw_rand_unif(lp->rng);
msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
msg->my_N_hop = 0;
......@@ -1291,8 +1275,48 @@ void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message
msg->intm_group_id = -1;
msg->intm_router_id = -1;
if(msg->packet_ID == TRACK)
printf("\x1B[34m-->Packet generated at terminal %d sending to router %d \x1b[0m\n", (int)lp->gid, s->router_id);
int target_queue = msg->rail_id;
int path_tie = 0;
if(s->params->rail_select == RAIL_PATH){
// Set starting rail to the first rail (rail 0)
target_queue = 0;
// Get information on destination terminal from its LP ID
codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name,
&mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id,
&mapping_offset);
// Get number of terminal LPs per repetition so we can calculate local/relative router ID for the destination terminal
int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM,
s->anno, 0);
// Compute relative id of router in the first rail that is connected to the destination terminal
int rail_one_dest_router_rel_id = (mapping_offset + (mapping_rep_id * num_lps)) / s->params->num_cn;
// Get path length from rail 1 source router to rail 1 dest router
int path_length_rail1 = get_path_length_from_terminal(s->router_id, rail_one_dest_router_rel_id, p);
// Compute relative id of router in the second rail connected to destination compute terminal
int rail_two_dest_router_rel_id = s->params->slim_total_routers - 1 - rail_one_dest_router_rel_id;
// Get path length from rail 2 src router to rail 2 dest router
int path_length_rail2 = get_path_length_from_terminal(s->params->slim_total_routers - 1 - s->router_id, rail_two_dest_router_rel_id, p);
// Compare rail path lengths
if( path_length_rail2 < path_length_rail1 ){
target_queue = 1;
}else if( path_length_rail2 == path_length_rail1 ){
// Set path tie so we can break the tie with the RAIL_CONGESTION method below
path_tie = 1;
}
}
if(s->params->rail_select == RAIL_CONGESTION || path_tie) {
target_queue = tw_rand_integer(lp->rng,0,p->ports_per_nic-1);
int curr_buffer = s->vc_occupancy[target_queue];
for(int j = 0; j < s->params->ports_per_nic; j++) {
int next = (msg->rail_id + j) % s->params->ports_per_nic;
if(s->vc_occupancy[next] < curr_buffer) {
curr_buffer = s->vc_occupancy[next];
target_queue = next;
}
}
}
msg->saved_vc = target_queue;
for(i = 0; i < num_chunks; i++)
{
......@@ -1320,28 +1344,33 @@ void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message
cur_chunk->msg.chunk_id = i;
append_to_terminal_message_list(s->terminal_msgs, s->terminal_msgs_tail,
0, cur_chunk);
s->terminal_length += s->params->chunk_size;
target_queue, cur_chunk);
s->terminal_length[target_queue] += s->params->chunk_size;
}
if(s->terminal_length < 2 * s->params->cn_vc_size)
{
model_net_method_idle_event(nic_ts, 0, lp);
}
else
{
bf->c11 = 1;
s->issueIdle = 1;
if(s->last_buf_full[0] == 0.0)
for(int j=0; j<s->params->ports_per_nic; j++){
if(s->terminal_length[j] < s->params->cn_vc_size)
{
bf->c8 = 1;
msg->saved_busy_time = s->last_buf_full[0];
/* TODO: Assumes a single vc from terminal to router */
s->last_buf_full[0] = tw_now(lp);
#if DEBUG
printf("(%lf) packet_generate() idle_event2 time offset:%lf\n",tw_now(lp),nic_ts);
#endif
model_net_method_idle_event2(nic_ts, 0, j, lp);
}
else
{
bf->c11 = 1;
s->issueIdle[j] = 1;
if(s->last_buf_full[j] == 0.0)
{
bf->c8 = 1;
msg->saved_busy_time = s->last_buf_full[j];
/* TODO: Assumes a single vc from terminal to router */
s->last_buf_full[j] = tw_now(lp);
}
}
}
if(s->in_send_loop == 0)
if(s->in_send_loop[target_queue] == 0)
{
bf->c5 = 1;
ts = codes_local_latency(lp);
......@@ -1349,8 +1378,9 @@ void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message
tw_event* e = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY,
(void**)&m, NULL);
m->type = T_SEND;
m->vc_index = target_queue;
m->magic = slim_terminal_magic_num;
s->in_send_loop = 1;
s->in_send_loop[target_queue] = 1;
tw_event_send(e);
}
......@@ -1370,23 +1400,23 @@ void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message
void slim_packet_send_rc(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
if(bf->c1) {
s->in_send_loop = 1;
s->in_send_loop[msg->vc_index] = 1;
if(bf->c10)
s->last_buf_full[0] = msg->saved_busy_time;
s->last_buf_full[msg->vc_index] = msg->saved_busy_time;
return;