Commit f61a0c6c authored by Misbah Mubarak's avatar Misbah Mubarak

First pass on the congestion control in torus model, fixing stat collection in dragonfly

parent 7a7092e1
...@@ -71,7 +71,8 @@ struct terminal_message ...@@ -71,7 +71,8 @@ struct terminal_message
/* for reverse computation */ /* for reverse computation */
short path_type; short path_type;
tw_stime saved_available_time; tw_stime saved_available_time;
tw_stime saved_credit_time; tw_stime saved_avg_time;
tw_stime saved_start_time;
tw_stime saved_collective_init_time; tw_stime saved_collective_init_time;
tw_stime saved_hist_start_time; tw_stime saved_hist_start_time;
tw_stime msg_start_time; tw_stime msg_start_time;
......
...@@ -40,8 +40,7 @@ struct nodes_message ...@@ -40,8 +40,7 @@ struct nodes_message
nodes_event_t type; nodes_event_t type;
/* for reverse computation */ /* for reverse computation */
int saved_src_dim; int saved_channel;
int saved_src_dir;
/* coordinates of the destination torus nodes */ /* coordinates of the destination torus nodes */
int* dest; int* dest;
...@@ -71,6 +70,9 @@ struct nodes_message ...@@ -71,6 +70,9 @@ struct nodes_message
/* for reverse computation of a node's fan in*/ /* for reverse computation of a node's fan in*/
int saved_fan_nodes; int saved_fan_nodes;
int source_channel;
int saved_queue;
/* chunk id of the flit (distinguishes flits) */ /* chunk id of the flit (distinguishes flits) */
int chunk_id; int chunk_id;
......
...@@ -14,7 +14,7 @@ PARAMS ...@@ -14,7 +14,7 @@ PARAMS
modelnet_order=( "dragonfly" ); modelnet_order=( "dragonfly" );
# scheduler options # scheduler options
modelnet_scheduler="fcfs"; modelnet_scheduler="fcfs";
chunk_size="64"; chunk_size="512";
# modelnet_scheduler="round-robin"; # modelnet_scheduler="round-robin";
num_vcs="1"; num_vcs="1";
num_routers="6"; num_routers="6";
...@@ -24,6 +24,6 @@ PARAMS ...@@ -24,6 +24,6 @@ PARAMS
local_bandwidth="5.25"; local_bandwidth="5.25";
global_bandwidth="4.7"; global_bandwidth="4.7";
cn_bandwidth="5.25"; cn_bandwidth="5.25";
message_size="528"; message_size="544";
routing="adaptive"; routing="adaptive";
} }
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
#include "codes/model-net.h" #include "codes/model-net.h"
#include "codes/rc-stack.h" #include "codes/rc-stack.h"
#define TRACE -1 #define TRACE 0
#define TRACK 0 #define TRACK 0
char workload_type[128]; char workload_type[128];
...@@ -344,7 +344,7 @@ static void mpi_queue_update(struct mpi_queue_ptrs* mpi_queue, struct codes_work ...@@ -344,7 +344,7 @@ static void mpi_queue_update(struct mpi_queue_ptrs* mpi_queue, struct codes_work
/* prints the elements of a queue (for debugging purposes). */ /* prints the elements of a queue (for debugging purposes). */
static void printCompletedQueue(nw_state* s, tw_lp* lp) static void printCompletedQueue(nw_state* s, tw_lp* lp)
{ {
if(TRACE == lp->gid) if(TRACE == s->nw_id)
{ {
printf("\n %lf contents of completed operations queue ", tw_now(lp)); printf("\n %lf contents of completed operations queue ", tw_now(lp));
struct completed_requests* current = s->completed_reqs; struct completed_requests* current = s->completed_reqs;
...@@ -370,7 +370,7 @@ static void notify_waits_rc(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, du ...@@ -370,7 +370,7 @@ static void notify_waits_rc(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, du
s->saved_pending_wait = NULL; s->saved_pending_wait = NULL;
} }
*/ */
if(lp->gid == TRACE) if(s->nw_id == TRACE)
printf("\n %lf reverse -- notify waits req id %d ", tw_now(lp), completed_req); printf("\n %lf reverse -- notify waits req id %d ", tw_now(lp), completed_req);
printCompletedQueue(s, lp); printCompletedQueue(s, lp);
...@@ -380,7 +380,7 @@ static void notify_waits_rc(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, du ...@@ -380,7 +380,7 @@ static void notify_waits_rc(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, du
/* if a wait-elem exists, it means the request ID has been matched*/ /* if a wait-elem exists, it means the request ID has been matched*/
if(m->u.rc.matched_op == 2) if(m->u.rc.matched_op == 2)
{ {
if(lp->gid == TRACE) if(s->nw_id == TRACE)
{ {
printf("\n %lf matched req id %d ", tw_now(lp), completed_req); printf("\n %lf matched req id %d ", tw_now(lp), completed_req);
printCompletedQueue(s, lp); printCompletedQueue(s, lp);
...@@ -409,7 +409,7 @@ static int notify_waits(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, dumpi_ ...@@ -409,7 +409,7 @@ static int notify_waits(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, dumpi_
struct pending_waits* wait_elem = s->pending_waits; struct pending_waits* wait_elem = s->pending_waits;
m->u.rc.matched_op = 0; m->u.rc.matched_op = 0;
if(lp->gid == TRACE) if(s->nw_id == TRACE)
printf("\n %lf notify waits req id %d ", tw_now(lp), completed_req); printf("\n %lf notify waits req id %d ", tw_now(lp), completed_req);
if(!wait_elem) if(!wait_elem)
...@@ -438,7 +438,7 @@ static int notify_waits(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, dumpi_ ...@@ -438,7 +438,7 @@ static int notify_waits(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, dumpi_
{ {
if(wait_elem->mpi_op->u.waits.req_ids[i] == completed_req) if(wait_elem->mpi_op->u.waits.req_ids[i] == completed_req)
{ {
if(lp->gid == TRACE) if(s->nw_id == TRACE)
printCompletedQueue(s, lp); printCompletedQueue(s, lp);
m->u.rc.matched_op = 1; m->u.rc.matched_op = 1;
wait_elem->num_completed++; wait_elem->num_completed++;
...@@ -447,7 +447,7 @@ static int notify_waits(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, dumpi_ ...@@ -447,7 +447,7 @@ static int notify_waits(nw_state* s, tw_bf* bf, tw_lp* lp, nw_message* m, dumpi_
if(wait_elem->num_completed == required_count) if(wait_elem->num_completed == required_count)
{ {
if(lp->gid == TRACE) if(s->nw_id == TRACE)
{ {
printf("\n %lf req %d completed %d", tw_now(lp), completed_req, wait_elem->num_completed); printf("\n %lf req %d completed %d", tw_now(lp), completed_req, wait_elem->num_completed);
printCompletedQueue(s, lp); printCompletedQueue(s, lp);
...@@ -513,7 +513,7 @@ static void codes_exec_mpi_wait(nw_state* s, tw_lp* lp, nw_message * m, struct c ...@@ -513,7 +513,7 @@ static void codes_exec_mpi_wait(nw_state* s, tw_lp* lp, nw_message * m, struct c
static void codes_exec_mpi_wait_all_rc(nw_state* s, nw_message* m, tw_lp* lp, struct codes_workload_op * mpi_op) static void codes_exec_mpi_wait_all_rc(nw_state* s, nw_message* m, tw_lp* lp, struct codes_workload_op * mpi_op)
{ {
if(lp->gid == TRACE) if(s->nw_id == TRACE)
{ {
printf("\n %lf codes exec mpi waitall reverse %d ", tw_now(lp), m->u.rc.found_match); printf("\n %lf codes exec mpi waitall reverse %d ", tw_now(lp), m->u.rc.found_match);
printCompletedQueue(s, lp); printCompletedQueue(s, lp);
...@@ -837,7 +837,6 @@ static void codes_exec_comp_delay( ...@@ -837,7 +837,6 @@ static void codes_exec_comp_delay(
msg->msg_type = MPI_OP_GET_NEXT; msg->msg_type = MPI_OP_GET_NEXT;
tw_event_send(e); tw_event_send(e);
} }
/* reverse computation operation for MPI irecv */ /* reverse computation operation for MPI irecv */
...@@ -1257,7 +1256,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l ...@@ -1257,7 +1256,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
s->nw_id, s->num_completed); s->nw_id, s->num_completed);
m->u.rc.saved_op = mpi_op; m->u.rc.saved_op = mpi_op;
if(mpi_op->op_type == CODES_WK_END) if(mpi_op->op_type == CODES_WK_END && s->num_completed == 50000)
{ {
//rc_stack_push(lp, mpi_op, free, s->st); //rc_stack_push(lp, mpi_op, free, s->st);
s->elapsed_time = tw_now(lp) - s->start_time; s->elapsed_time = tw_now(lp) - s->start_time;
......
...@@ -147,7 +147,6 @@ struct terminal_state ...@@ -147,7 +147,6 @@ struct terminal_state
int* vc_occupancy; // NUM_VC int* vc_occupancy; // NUM_VC
int num_vcs; int num_vcs;
tw_stime terminal_available_time; tw_stime terminal_available_time;
tw_stime next_credit_available_time;
terminal_message_list **terminal_msgs; terminal_message_list **terminal_msgs;
terminal_message_list **terminal_msgs_tail; terminal_message_list **terminal_msgs_tail;
int in_send_loop; int in_send_loop;
...@@ -248,7 +247,6 @@ struct router_state ...@@ -248,7 +247,6 @@ struct router_state
int* global_channel; int* global_channel;
tw_stime* next_output_available_time; tw_stime* next_output_available_time;
tw_stime* next_credit_available_time;
tw_stime* cur_hist_start_time; tw_stime* cur_hist_start_time;
terminal_message_list ***pending_msgs; terminal_message_list ***pending_msgs;
terminal_message_list ***pending_msgs_tail; terminal_message_list ***pending_msgs_tail;
...@@ -739,7 +737,6 @@ void router_setup(router_state * r, tw_lp * lp) ...@@ -739,7 +737,6 @@ void router_setup(router_state * r, tw_lp * lp)
r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int)); r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int));
r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime)); r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
r->next_credit_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
r->cur_hist_start_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime)); r->cur_hist_start_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
r->link_traffic = (int*)malloc(p->radix * sizeof(int)); r->link_traffic = (int*)malloc(p->radix * sizeof(int));
r->cur_hist_num = (int*)malloc(p->radix * sizeof(int)); r->cur_hist_num = (int*)malloc(p->radix * sizeof(int));
...@@ -760,7 +757,6 @@ void router_setup(router_state * r, tw_lp * lp) ...@@ -760,7 +757,6 @@ void router_setup(router_state * r, tw_lp * lp)
{ {
// Set credit & router occupancy // Set credit & router occupancy
r->next_output_available_time[i]=0; r->next_output_available_time[i]=0;
r->next_credit_available_time[i]=0;
r->cur_hist_start_time[i] = 0; r->cur_hist_start_time[i] = 0;
r->link_traffic[i]=0; r->link_traffic[i]=0;
r->cur_hist_num[i] = 0; r->cur_hist_num[i] = 0;
...@@ -1256,7 +1252,6 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -1256,7 +1252,6 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw
if(msg->path_type == NON_MINIMAL) if(msg->path_type == NON_MINIMAL)
nonmin_count--; nonmin_count--;
s->next_credit_available_time = msg->saved_credit_time;
uint64_t num_chunks = msg->packet_size / s->params->chunk_size; uint64_t num_chunks = msg->packet_size / s->params->chunk_size;
N_finished_chunks--; N_finished_chunks--;
...@@ -1296,8 +1291,6 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -1296,8 +1291,6 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw
hash_link = qhash_search(s->rank_tbl, &key); hash_link = qhash_search(s->rank_tbl, &key);
tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link); tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link);
total_hops -= msg->my_N_hop;
mn_stats* stat; mn_stats* stat;
stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
if(bf->c1) if(bf->c1)
...@@ -1312,16 +1305,16 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -1312,16 +1305,16 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw
dragonfly_max_latency = msg->saved_available_time; dragonfly_max_latency = msg->saved_available_time;
if(bf->c7) if(!hash_link)
{ {
s->finished_msgs--; s->finished_msgs--;
total_msg_sz -= msg->total_size; total_msg_sz -= msg->total_size;
N_finished_msgs--; N_finished_msgs--;
stat->recv_time -= tw_now(lp) - msg->travel_start_time; stat->recv_time -= msg->saved_start_time;
s->total_msg_time -= (tw_now(lp) - msg->msg_start_time); s->total_msg_time -= msg->saved_start_time;
s->total_msg_size -= msg->total_size; s->total_msg_size -= msg->total_size;
dragonfly_total_time -= (tw_now(lp) - msg->travel_start_time); dragonfly_total_time -= msg->saved_avg_time;
struct dfly_qhash_entry * d_entry_pop = (struct dfly_qhash_entry*)rc_stack_pop(s->st); struct dfly_qhash_entry * d_entry_pop = (struct dfly_qhash_entry*)rc_stack_pop(s->st);
qhash_add(s->rank_tbl, &key, &(d_entry_pop->hash_link)); qhash_add(s->rank_tbl, &key, &(d_entry_pop->hash_link));
...@@ -1354,7 +1347,7 @@ void send_remote_event(terminal_state * s, terminal_message * msg, tw_lp * lp, t ...@@ -1354,7 +1347,7 @@ void send_remote_event(terminal_state * s, terminal_message * msg, tw_lp * lp, t
model_net_set_msg_param(MN_MSG_PARAM_START_TIME, MN_MSG_PARAM_START_TIME_VAL, &(msg->msg_start_time)); model_net_set_msg_param(MN_MSG_PARAM_START_TIME, MN_MSG_PARAM_START_TIME_VAL, &(msg->msg_start_time));
model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category, msg->event_rc = model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category,
msg->sender_lp, msg->pull_size, ts, msg->sender_lp, msg->pull_size, ts,
remote_event_size, tmp_ptr, 0, NULL, lp); remote_event_size, tmp_ptr, 0, NULL, lp);
} }
...@@ -1539,10 +1532,12 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1539,10 +1532,12 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg,
N_finished_msgs++; N_finished_msgs++;
total_msg_sz += msg->total_size; total_msg_sz += msg->total_size;
dragonfly_total_time += tw_now( lp ) - msg->travel_start_time; msg->saved_avg_time = tw_now( lp ) - msg->travel_start_time;
stat->recv_time += tw_now(lp) - msg->travel_start_time; dragonfly_total_time += msg->saved_avg_time;
msg->saved_start_time = (tw_now(lp) - msg->msg_start_time);
stat->recv_time += msg->saved_start_time;
s->finished_msgs++; s->finished_msgs++;
s->total_msg_time += (tw_now(lp) - msg->msg_start_time); s->total_msg_time += msg->saved_start_time;
s->total_msg_size += msg->total_size; s->total_msg_size += msg->total_size;
if (dragonfly_max_latency < tw_now( lp ) - msg->travel_start_time) { if (dragonfly_max_latency < tw_now( lp ) - msg->travel_start_time) {
...@@ -1923,7 +1918,7 @@ dragonfly_terminal_final( terminal_state * s, ...@@ -1923,7 +1918,7 @@ dragonfly_terminal_final( terminal_state * s,
int written = 0; int written = 0;
if(!s->terminal_id) if(!s->terminal_id)
written = sprintf(s->output_buf, "# Format <LP id> <Terminal ID> <Avg Msg Size> <Avg Msg Time> <# Msgs finished>"); written = sprintf(s->output_buf, "# Format <LP id> <Terminal ID> <Total Msg Size> <Total Msg Time> <# Msgs finished>\n");
written += sprintf(s->output_buf + written, "%lu %lu %ld %lf %ld %ld\n", lp->gid, s->terminal_id, s->total_msg_size, s->total_msg_time, s->finished_msgs, s->finished_packets); written += sprintf(s->output_buf + written, "%lu %lu %ld %lf %ld %ld\n", lp->gid, s->terminal_id, s->total_msg_size, s->total_msg_time, s->finished_msgs, s->finished_packets);
lp_io_write(lp->gid, "dragonfly-msg-stats", written, s->output_buf); lp_io_write(lp->gid, "dragonfly-msg-stats", written, s->output_buf);
...@@ -2042,9 +2037,7 @@ get_next_stop(router_state * s, ...@@ -2042,9 +2037,7 @@ get_next_stop(router_state * s,
if(msg->last_hop == TERMINAL && path == NON_MINIMAL) if(msg->last_hop == TERMINAL && path == NON_MINIMAL)
{ {
if(dest_group_id != s->group_id) if(dest_group_id != s->group_id)
{
msg->intm_group_id = intm_id; msg->intm_group_id = intm_id;
}
} }
/******************** DECIDE THE DESTINATION GROUP ***********************/ /******************** DECIDE THE DESTINATION GROUP ***********************/
/* It means that the packet has arrived at the inter-mediate group for non-minimal routing. Reset the group now. */ /* It means that the packet has arrived at the inter-mediate group for non-minimal routing. Reset the group now. */
...@@ -2200,7 +2193,7 @@ static int do_adaptive_routing( router_state * s, ...@@ -2200,7 +2193,7 @@ static int do_adaptive_routing( router_state * s,
assert(num_nonmin_hops <= 6); assert(num_nonmin_hops <= 6);
/* average the local queues of the router */ /* average the local queues of the router */
unsigned int q_avg = 0; /*unsigned int q_avg = 0;
int i; int i;
for( i = 0; i < s->params->radix; i++) for( i = 0; i < s->params->radix; i++)
{ {
...@@ -2209,7 +2202,7 @@ static int do_adaptive_routing( router_state * s, ...@@ -2209,7 +2202,7 @@ static int do_adaptive_routing( router_state * s,
s->vc_occupancy[i][2]; s->vc_occupancy[i][2];
} }
q_avg = q_avg / (s->params->radix - 1); q_avg = q_avg / (s->params->radix - 1);
*/
//int min_out_chan = minimal_out_port; //int min_out_chan = minimal_out_port;
//int nonmin_out_chan = nonmin_out_port; //int nonmin_out_chan = nonmin_out_port;
...@@ -2311,15 +2304,13 @@ router_packet_receive( router_state * s, ...@@ -2311,15 +2304,13 @@ router_packet_receive( router_state * s,
} else if(msg->last_hop == TERMINAL && routing == ADAPTIVE) { } else if(msg->last_hop == TERMINAL && routing == ADAPTIVE) {
next_stop = do_adaptive_routing(s, bf, msg, lp, dest_router_id, intm_id); next_stop = do_adaptive_routing(s, bf, msg, lp, dest_router_id, intm_id);
} else { } else {
if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
assert(msg->path_type == MINIMAL || msg->path_type == NON_MINIMAL);
if(routing == MINIMAL || routing == NON_MINIMAL) if(routing == MINIMAL || routing == NON_MINIMAL)
msg->path_type = routing; /*defaults to the routing algorithm if we msg->path_type = routing; /*defaults to the routing algorithm if we
don't have adaptive routing here*/ don't have adaptive routing here*/
next_stop = get_next_stop(s, bf, msg, lp, msg->path_type, dest_router_id, next_stop = get_next_stop(s, bf, msg, lp, msg->path_type, dest_router_id,
intm_id); intm_id);
} }
assert(msg->path_type == MINIMAL || msg->path_type == NON_MINIMAL);
terminal_message_list * cur_chunk = (terminal_message_list *)malloc( terminal_message_list * cur_chunk = (terminal_message_list *)malloc(
sizeof(terminal_message_list)); sizeof(terminal_message_list));
init_terminal_message_list(cur_chunk, msg); init_terminal_message_list(cur_chunk, msg);
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment