Commit bf9dbce4 authored by Misbah Mubarak's avatar Misbah Mubarak

updates to MPI Sim Layer and dragonfly model

parent de244764
...@@ -25,5 +25,5 @@ PARAMS ...@@ -25,5 +25,5 @@ PARAMS
global_bandwidth="4.7"; global_bandwidth="4.7";
cn_bandwidth="5.25"; cn_bandwidth="5.25";
message_size="528"; message_size="528";
routing="minimal"; routing="adaptive";
} }
...@@ -157,6 +157,7 @@ struct nw_message ...@@ -157,6 +157,7 @@ struct nw_message
short matched_op; short matched_op;
dumpi_req_id saved_matched_req; dumpi_req_id saved_matched_req;
struct codes_workload_op* ptr_match_op; struct codes_workload_op* ptr_match_op;
struct codes_workload_op* saved_op;
struct pending_waits* saved_pending_wait; struct pending_waits* saved_pending_wait;
double saved_send_time; double saved_send_time;
...@@ -474,7 +475,7 @@ static void codes_exec_mpi_wait_rc(nw_state* s, nw_message* m, tw_lp* lp, struct ...@@ -474,7 +475,7 @@ static void codes_exec_mpi_wait_rc(nw_state* s, nw_message* m, tw_lp* lp, struct
{ {
mpi_completed_queue_insert_op(&s->completed_reqs, mpi_op->u.wait.req_id); mpi_completed_queue_insert_op(&s->completed_reqs, mpi_op->u.wait.req_id);
tw_rand_reverse_unif(lp->rng); tw_rand_reverse_unif(lp->rng);
rc_stack_pop(s->st); //rc_stack_pop(s->st);
} }
} }
...@@ -503,7 +504,7 @@ static void codes_exec_mpi_wait(nw_state* s, tw_lp* lp, nw_message * m, struct c ...@@ -503,7 +504,7 @@ static void codes_exec_mpi_wait(nw_state* s, tw_lp* lp, nw_message * m, struct c
wait_op->start_time = tw_now(lp); wait_op->start_time = tw_now(lp);
s->pending_waits = wait_op; s->pending_waits = wait_op;
rc_stack_push(lp, wait_op, free, s->st); //rc_stack_push(lp, wait_op, free, s->st);
} }
static void codes_exec_mpi_wait_all_rc(nw_state* s, nw_message* m, tw_lp* lp, struct codes_workload_op * mpi_op) static void codes_exec_mpi_wait_all_rc(nw_state* s, nw_message* m, tw_lp* lp, struct codes_workload_op * mpi_op)
...@@ -529,7 +530,7 @@ static void codes_exec_mpi_wait_all_rc(nw_state* s, nw_message* m, tw_lp* lp, st ...@@ -529,7 +530,7 @@ static void codes_exec_mpi_wait_all_rc(nw_state* s, nw_message* m, tw_lp* lp, st
else else
{ {
struct pending_waits* wait_op = s->pending_waits; struct pending_waits* wait_op = s->pending_waits;
rc_stack_pop(s->st); //rc_stack_pop(s->st);
s->pending_waits = NULL; s->pending_waits = NULL;
assert(!s->pending_waits); assert(!s->pending_waits);
if(lp->gid == TRACE) if(lp->gid == TRACE)
...@@ -583,7 +584,7 @@ static void codes_exec_mpi_wait_all( ...@@ -583,7 +584,7 @@ static void codes_exec_mpi_wait_all(
wait_op->mpi_op = mpi_op; wait_op->mpi_op = mpi_op;
wait_op->num_completed = num_completed; wait_op->num_completed = num_completed;
wait_op->start_time = tw_now(lp); wait_op->start_time = tw_now(lp);
rc_stack_push(lp, wait_op, free, s->st); //rc_stack_push(lp, wait_op, free, s->st);
s->pending_waits = wait_op; s->pending_waits = wait_op;
} }
} }
...@@ -997,7 +998,7 @@ static void update_arrival_queue_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_ ...@@ -997,7 +998,7 @@ static void update_arrival_queue_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_
s->recv_time = m->u.rc.saved_recv_time; s->recv_time = m->u.rc.saved_recv_time;
codes_local_latency_reverse(lp); codes_local_latency_reverse(lp);
rc_stack_pop(s->st); //rc_stack_pop(s->st);
if(m->u.rc.found_match >= 0) if(m->u.rc.found_match >= 0)
{ {
...@@ -1052,7 +1053,7 @@ static void update_arrival_queue(nw_state* s, tw_bf * bf, nw_message * m, tw_lp ...@@ -1052,7 +1053,7 @@ static void update_arrival_queue(nw_state* s, tw_bf * bf, nw_message * m, tw_lp
arrived_op->u.send.num_bytes = m->u.msg_info.num_bytes; arrived_op->u.send.num_bytes = m->u.msg_info.num_bytes;
arrived_op->u.send.tag = m->u.msg_info.tag; arrived_op->u.send.tag = m->u.msg_info.tag;
arrived_op->u.send.req_id = m->u.msg_info.req_id; arrived_op->u.send.req_id = m->u.msg_info.req_id;
rc_stack_push(lp, arrived_op, free, s->st); //rc_stack_push(lp, arrived_op, free, s->st);
int found_matching_recv = mpi_queue_remove_matching_op(s, lp, m, s->pending_recvs_queue, arrived_op); int found_matching_recv = mpi_queue_remove_matching_op(s, lp, m, s->pending_recvs_queue, arrived_op);
...@@ -1163,8 +1164,8 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) ...@@ -1163,8 +1164,8 @@ void nw_test_event_handler(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp) static void get_next_mpi_operation_rc(nw_state* s, tw_bf * bf, nw_message * m, tw_lp * lp)
{ {
struct codes_workload_op * mpi_op = struct codes_workload_op * mpi_op = m->u.rc.saved_op;
(struct codes_workload_op *)rc_stack_pop(s->st); //(struct codes_workload_op *)rc_stack_pop(s->st);
codes_workload_get_next_rc(wrkld_id, 0, (int)s->nw_id, mpi_op); codes_workload_get_next_rc(wrkld_id, 0, (int)s->nw_id, mpi_op);
...@@ -1244,10 +1245,11 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l ...@@ -1244,10 +1245,11 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
{ {
struct codes_workload_op * mpi_op = malloc(sizeof(struct codes_workload_op)); struct codes_workload_op * mpi_op = malloc(sizeof(struct codes_workload_op));
codes_workload_get_next(wrkld_id, 0, (int)s->nw_id, mpi_op); codes_workload_get_next(wrkld_id, 0, (int)s->nw_id, mpi_op);
rc_stack_push(lp, mpi_op, free, s->st);
m->u.rc.saved_op = mpi_op;
if(mpi_op->op_type == CODES_WK_END) if(mpi_op->op_type == CODES_WK_END)
{ {
//rc_stack_push(lp, mpi_op, free, s->st);
s->elapsed_time = tw_now(lp) - s->start_time; s->elapsed_time = tw_now(lp) - s->start_time;
return; return;
} }
...@@ -1306,6 +1308,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l ...@@ -1306,6 +1308,7 @@ static void get_next_mpi_operation(nw_state* s, tw_bf * bf, nw_message * m, tw_l
default: default:
printf("\n Invalid op type %d ", mpi_op->op_type); printf("\n Invalid op type %d ", mpi_op->op_type);
} }
//rc_stack_push(lp, mpi_op, free, s->st);
return; return;
} }
......
...@@ -177,6 +177,8 @@ struct terminal_state ...@@ -177,6 +177,8 @@ struct terminal_state
short is_leaf; short is_leaf;
struct rc_stack * st; struct rc_stack * st;
int issueIdle;
int terminal_length;
/* to maintain a count of child nodes that have fanned in at the parent during the collective /* to maintain a count of child nodes that have fanned in at the parent during the collective
fan-in phase*/ fan-in phase*/
...@@ -693,7 +695,9 @@ terminal_init( terminal_state * s, ...@@ -693,7 +695,9 @@ terminal_init( terminal_state * s,
(terminal_message_list**)malloc(1*sizeof(terminal_message_list*)); (terminal_message_list**)malloc(1*sizeof(terminal_message_list*));
s->terminal_msgs[0] = NULL; s->terminal_msgs[0] = NULL;
s->terminal_msgs_tail[0] = NULL; s->terminal_msgs_tail[0] = NULL;
s->terminal_length = 0;
s->in_send_loop = 0; s->in_send_loop = 0;
s->issueIdle = 0;
dragonfly_collective_init(s, lp); dragonfly_collective_init(s, lp);
return; return;
...@@ -983,6 +987,7 @@ void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -983,6 +987,7 @@ void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
term_rev_ecount++; term_rev_ecount++;
term_ecount--; term_ecount--;
tw_rand_reverse_unif(lp->rng);
int num_chunks = msg->packet_size/s->params->chunk_size; int num_chunks = msg->packet_size/s->params->chunk_size;
if(msg->packet_size % s->params->chunk_size) if(msg->packet_size % s->params->chunk_size)
...@@ -1000,6 +1005,9 @@ void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1000,6 +1005,9 @@ void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
codes_local_latency_reverse(lp); codes_local_latency_reverse(lp);
s->in_send_loop = 0; s->in_send_loop = 0;
} }
if(bf->c11) {
s->issueIdle = 0;
}
struct mn_stats* stat; struct mn_stats* stat;
stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
stat->send_count--; stat->send_count--;
...@@ -1012,12 +1020,11 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1012,12 +1020,11 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg,
tw_lp * lp) { tw_lp * lp) {
term_ecount++; term_ecount++;
tw_stime ts; tw_stime ts, nic_ts;
assert(lp->gid != msg->dest_terminal_id); assert(lp->gid != msg->dest_terminal_id);
const dragonfly_param *p = s->params; const dragonfly_param *p = s->params;
int i, total_event_size; int i, total_event_size;
int num_chunks = msg->packet_size / p->chunk_size; int num_chunks = msg->packet_size / p->chunk_size;
if (msg->packet_size % s->params->chunk_size) if (msg->packet_size % s->params->chunk_size)
...@@ -1026,6 +1033,8 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1026,6 +1033,8 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg,
if(!num_chunks) if(!num_chunks)
num_chunks = 1; num_chunks = 1;
nic_ts = g_tw_lookahead + s->params->cn_delay * msg->packet_size + tw_rand_unif(lp->rng);
msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter; msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
msg->my_N_hop = 0; msg->my_N_hop = 0;
msg->my_l_hop = 0; msg->my_l_hop = 0;
...@@ -1059,8 +1068,15 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1059,8 +1068,15 @@ void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg,
cur_chunk->msg.chunk_id = i; cur_chunk->msg.chunk_id = i;
append_to_terminal_message_list(s->terminal_msgs, s->terminal_msgs_tail, append_to_terminal_message_list(s->terminal_msgs, s->terminal_msgs_tail,
0, cur_chunk); 0, cur_chunk);
s->terminal_length += s->params->chunk_size;
} }
if(s->terminal_length < 2 * s->params->cn_vc_size) {
model_net_method_idle_event(nic_ts, 0, lp);
} else {
bf->c11 = 1;
s->issueIdle = 1;
}
if(s->in_send_loop == 0) { if(s->in_send_loop == 0) {
bf->c5 = 1; bf->c5 = 1;
...@@ -1116,8 +1132,11 @@ void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1116,8 +1132,11 @@ void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
s->in_send_loop = 1; s->in_send_loop = 1;
} }
if(bf->c5) if(bf->c5)
{
codes_local_latency_reverse(lp); codes_local_latency_reverse(lp);
return; s->issueIdle = 1;
}
return;
} }
/* sends the packet from the current dragonfly compute node to the attached router */ /* sends the packet from the current dragonfly compute node to the attached router */
void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg,
...@@ -1212,10 +1231,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1212,10 +1231,10 @@ void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg,
bf->c4 = 1; bf->c4 = 1;
s->in_send_loop = 0; s->in_send_loop = 0;
} }
if(cur_entry == NULL && s->vc_occupancy[0] + s->params->chunk_size <= s->params->cn_vc_size) if(s->issueIdle) {
{ bf->c5 = 1;
bf->c5 = 1; s->issueIdle = 0;
model_net_method_idle_event(codes_local_latency(lp), 0, lp); model_net_method_idle_event(codes_local_latency(lp), 0, lp);
} }
return; return;
} }
...@@ -1232,7 +1251,34 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -1232,7 +1251,34 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw
if(msg->path_type == NON_MINIMAL) if(msg->path_type == NON_MINIMAL)
nonmin_count--; nonmin_count--;
struct qhash_head * hash_link = NULL; s->next_credit_available_time = msg->saved_credit_time;
uint64_t num_chunks = msg->packet_size / s->params->chunk_size;
if(msg->chunk_id == num_chunks - 1)
{
mn_stats* stat;
stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
stat->recv_count--;
stat->recv_bytes -= msg->packet_size;
stat->recv_time -= tw_now(lp) - msg->travel_start_time;
total_hops -= msg->my_N_hop;
N_finished_packets--;
dragonfly_total_time -= (tw_now(lp) - msg->travel_start_time);
if(bf->c3)
dragonfly_max_latency = msg->saved_available_time;
}
if (msg->chunk_id == num_chunks-1 &&
msg->remote_event_size_bytes &&
msg->is_pull)
{
int net_id = model_net_get_id(LP_METHOD_NM);
model_net_event_rc(net_id, lp, msg->pull_size);
}
/*struct qhash_head * hash_link = NULL;
struct dfly_qhash_entry * tmp = NULL; struct dfly_qhash_entry * tmp = NULL;
struct dfly_hash_key key; struct dfly_hash_key key;
...@@ -1280,7 +1326,7 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw ...@@ -1280,7 +1326,7 @@ void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw
assert(tmp); assert(tmp);
tmp->num_chunks--; tmp->num_chunks--;
*/
return; return;
} }
void send_remote_event(terminal_state * s, terminal_message * msg, tw_lp * lp, tw_bf * bf, char * event_data, int remote_event_size) void send_remote_event(terminal_state * s, terminal_message * msg, tw_lp * lp, tw_bf * bf, char * event_data, int remote_event_size)
...@@ -1388,9 +1434,50 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1388,9 +1434,50 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg,
} }
#endif #endif
tw_event * e;
terminal_message * m;
if(msg->chunk_id == num_chunks-1)
{
bf->c2 = 1;
mn_stats* stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
stat->recv_count++;
stat->recv_bytes += msg->packet_size;
stat->recv_time += tw_now(lp) - msg->travel_start_time;
N_finished_packets++;
total_hops -= msg->my_N_hop;
dragonfly_total_time += tw_now( lp ) - msg->travel_start_time;
if (dragonfly_max_latency < tw_now( lp ) - msg->travel_start_time)
{
bf->c3 = 1;
msg->saved_available_time = dragonfly_max_latency;
dragonfly_max_latency=tw_now( lp ) - msg->travel_start_time;
}
if(msg->remote_event_size_bytes)
{
void * tmp_ptr = model_net_method_get_edata(DRAGONFLY, msg);
ts = g_tw_lookahead + 0.1 + (1/s->params->cn_bandwidth) * msg->remote_event_size_bytes;
if (msg->is_pull){
struct codes_mctx mc_dst =
codes_mctx_set_global_direct(msg->sender_mn_lp);
struct codes_mctx mc_src =
codes_mctx_set_global_direct(lp->gid);
int net_id = model_net_get_id(LP_METHOD_NM);
model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category,
msg->sender_lp, msg->pull_size, ts,
msg->remote_event_size_bytes, tmp_ptr, 0, NULL, lp);
}
else {
e = tw_event_new(msg->final_dest_gid, ts, lp);
m = tw_event_data(e);
memcpy(m, tmp_ptr, msg->remote_event_size_bytes);
tw_event_send(e);
}
}
}
/* Now retreieve the number of chunks completed from the hash and update /* Now retreieve the number of chunks completed from the hash and update
* them */ * them */
void *m_data_src = model_net_method_get_edata(DRAGONFLY, msg); /*void *m_data_src = model_net_method_get_edata(DRAGONFLY, msg);
struct qhash_head *hash_link = NULL; struct qhash_head *hash_link = NULL;
struct dfly_qhash_entry * tmp = NULL; struct dfly_qhash_entry * tmp = NULL;
...@@ -1401,8 +1488,8 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1401,8 +1488,8 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg,
hash_link = qhash_search(s->rank_tbl, &key); hash_link = qhash_search(s->rank_tbl, &key);
tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link); tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link);
/* If an entry does not exist then create one */ *//* If an entry does not exist then create one */
if(!hash_link) /*if(!hash_link)
{ {
bf->c5 = 1; bf->c5 = 1;
struct dfly_qhash_entry * d_entry = malloc(sizeof (struct dfly_qhash_entry)); struct dfly_qhash_entry * d_entry = malloc(sizeof (struct dfly_qhash_entry));
...@@ -1422,8 +1509,8 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1422,8 +1509,8 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg,
if(tmp->num_chunks >= total_chunks) if(tmp->num_chunks >= total_chunks)
is_last_msg = 1; is_last_msg = 1;
/* if its the last chunk of the packet then handle the remote event data */ *//* if its the last chunk of the packet then handle the remote event data */
if(msg->chunk_id == num_chunks - 1) /*if(msg->chunk_id == num_chunks - 1)
{ {
bf->c1 = 1; bf->c1 = 1;
mn_stats* stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); mn_stats* stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
...@@ -1445,15 +1532,15 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1445,15 +1532,15 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg,
} }
if(msg->remote_event_size_bytes > 0 && !tmp->remote_event_data) if(msg->remote_event_size_bytes > 0 && !tmp->remote_event_data)
{ {
/* Retreive the remote event entry */ *//* Retreive the remote event entry */
tmp->remote_event_data = (void*)malloc(msg->remote_event_size_bytes); /*tmp->remote_event_data = (void*)malloc(msg->remote_event_size_bytes);
assert(tmp->remote_event_data); assert(tmp->remote_event_data);
tmp->remote_event_size = msg->remote_event_size_bytes; tmp->remote_event_size = msg->remote_event_size_bytes;
memcpy(tmp->remote_event_data, m_data_src, msg->remote_event_size_bytes); memcpy(tmp->remote_event_data, m_data_src, msg->remote_event_size_bytes);
} }*/
/* If all chunks of a message have arrived then send a remote event to the /* If all chunks of a message have arrived then send a remote event to the
* callee*/ * callee*/
if(is_last_msg) /*if(is_last_msg)
{ {
bf->c7 = 1; bf->c7 = 1;
s->finished_msgs++; s->finished_msgs++;
...@@ -1469,11 +1556,11 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, ...@@ -1469,11 +1556,11 @@ void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg,
void *m_data = model_net_method_get_edata(DRAGONFLY, msg); void *m_data = model_net_method_get_edata(DRAGONFLY, msg);
send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size); send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size);
} }
/* Remove the hash entry */ *//* Remove the hash entry */
qhash_del(hash_link); /*qhash_del(hash_link);
rc_stack_push(lp, tmp, free_tmp, s->st); rc_stack_push(lp, tmp, free_tmp, s->st);
s->rank_tbl_pop--; s->rank_tbl_pop--;
} }*/
return; return;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment