Commit 9fbdc45a authored by Misbah Mubarak's avatar Misbah Mubarak
Browse files

Merge branch 'damaris' into 'master'

ROSS-Damaris integration support

See merge request !54
parents 621ba97b d271ea94
......@@ -88,3 +88,7 @@ endif
LDADD += ${DUMPI_LIBS}
endif
if USE_RDAMARIS
AM_CPPFLAGS += ${ROSS_Damaris_CFLAGS} -DUSE_RDAMARIS=1
LDADD += ${ROSS_Damaris_LIBS}
endif
......@@ -97,6 +97,8 @@ static inline void codes_local_latency_reverse(tw_lp *lp)
return;
}
void codes_comm_update();
#ifdef __cplusplus
}
#endif
......
......@@ -92,6 +92,8 @@ struct terminal_plus_message
tw_stime saved_total_time;
tw_stime saved_sample_time;
tw_stime msg_start_time;
tw_stime saved_busy_time_ross;
tw_stime saved_fin_chunks_ross;
};
#ifdef __cplusplus
......
......@@ -89,6 +89,16 @@ AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)],
PKG_CHECK_MODULES_STATIC([ROSS], [ross], [],
[AC_MSG_ERROR([Could not find working ross installation via pkg-config])])
#check for Damaris
AC_ARG_WITH([damaris],[AS_HELP_STRING([--with-damaris],
[build with ROSS-Damaris in situ analysis support])],
[use_rdamaris=yes],[use_rdamaris=no])
if test "x${use_rdamaris}" = xyes ; then
PKG_CHECK_MODULES_STATIC([ROSS_Damaris], [ross-damaris], [],
[AC_MSG_ERROR([Could not find working ROSS-Damaris installation via pkg-config])])
fi
AM_CONDITIONAL(USE_RDAMARIS, [test "x${use_rdamaris}" = xyes])
# check for enable-g
AC_ARG_ENABLE([g],[AS_HELP_STRING([--enable-g],
[Build with GDB symbols])],
......
......@@ -23,6 +23,6 @@ PARAMS
local_bandwidth="5.25";
global_bandwidth="4.7";
cn_bandwidth="5.25";
message_size="592";
message_size="608";
routing="adaptive";
}
......@@ -414,6 +414,11 @@ int main( int argc, char** argv )
workload_type[0]='\0';
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(strlen(workload_file) == 0)
{
......@@ -423,10 +428,10 @@ int main( int argc, char** argv )
return -1;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
nw_add_lp_type();
......@@ -446,20 +451,20 @@ int main( int argc, char** argv )
double total_avg_comp_time;
long overall_sends, overall_recvs, overall_waits, overall_cols;
MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_col_time, &avg_col_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_compute_time, &total_avg_comp_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_sends, &overall_sends, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_recvs, &overall_recvs, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_waits, &overall_waits, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_collectives, &overall_cols, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_col_time, &avg_col_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_compute_time, &total_avg_comp_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_sends, &overall_sends, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_recvs, &overall_recvs, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_waits, &overall_waits, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_collectives, &overall_cols, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
if(!g_tw_mynode)
printf("\n Total bytes sent %lld recvd %lld \n avg runtime %lf \n avg comm time %lf avg compute time %lf \n avg send time %lf \n avg recv time %lf \n avg wait time %lf \n total sends %ld total recvs %ld total waits %ld total collectives %ld ", total_bytes_sent, total_bytes_recvd,
......@@ -470,6 +475,9 @@ int main( int argc, char** argv )
total_avg_recv_time/num_net_lps,
total_avg_wait_time/num_net_lps,
overall_sends, overall_recvs, overall_waits, overall_cols);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
......
......@@ -2725,6 +2725,11 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
workload_type[0]='\0';
tw_opt_add(app_opt);
tw_init(argc, argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(strcmp(workload_type, "dumpi") != 0 && strcmp(workload_type, "online") != 0)
{
......@@ -2909,7 +2914,7 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
MPI_Reduce(&max_recv_time, &total_max_recv_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_syn_data, &g_total_syn_data, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_syn_data, &g_total_syn_data, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
assert(num_net_traces);
......@@ -2943,6 +2948,9 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
if(alloc_spec)
codes_jobmap_destroy(jobmap_ctx);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
......
......@@ -453,6 +453,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -516,6 +521,9 @@ int main(
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
}
......
......@@ -280,7 +280,7 @@ static void handle_kickoff_event(
// codes_mapping_get_lp_id(group_name, lp_type_name, anno, 1, local_dest / num_servers_per_rep, local_dest % num_servers_per_rep, &global_dest);
global_dest = codes_mapping_get_lpid_from_relative(local_dest, group_name, lp_type_name, NULL, 0);
ns->msg_sent_count++;
model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
m->event_rc = model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
issue_event(ns, lp);
return;
......@@ -416,6 +416,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -477,6 +482,9 @@ int main(
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
}
......
......@@ -436,6 +436,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
offset = 1;
......@@ -446,10 +451,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
......@@ -484,7 +489,7 @@ int main(
printf("num_nodes:%d \n",num_nodes);
if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0)
if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_CODES) < 0)
{
return(-1);
}
......@@ -516,11 +521,13 @@ int main(
}
#endif
if(lp_io_flush(handle, MPI_COMM_WORLD) < 0)
if(lp_io_flush(handle, MPI_COMM_CODES) < 0)
{
return(-1);
}
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
#if PARAMS_LOG
......
......@@ -494,6 +494,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -502,10 +507,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
svr_add_lp_type();
......@@ -526,7 +531,7 @@ int main(
num_nodes_per_grp = num_routers_per_grp * num_servers_per_rep;
total_routers = num_routers_per_grp * num_routers_per_grp * 2;
/* if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0)
/* if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_CODES) < 0)
{
return(-1);
}
......@@ -536,7 +541,7 @@ int main(
{
do_lp_io = 1;
int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_prepare failure");
}
......@@ -558,7 +563,7 @@ int main(
if (do_lp_io){
int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_flush failure");
}
......@@ -578,12 +583,15 @@ int main(
#endif
}
/* if(lp_io_flush(handle, MPI_COMM_WORLD) < 0)
/* if(lp_io_flush(handle, MPI_COMM_CODES) < 0)
{
assert(ret == 0 || !"lp_io_flush failure");
return(-1);
}
*/
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
if(rank == 0)
......
......@@ -407,6 +407,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -415,10 +420,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
svr_add_lp_type();
......@@ -457,15 +462,18 @@ int main(
{
do_lp_io = 1;
int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_prepare failure");
}
tw_run();
if (do_lp_io){
int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
}
......
......@@ -380,7 +380,7 @@ struct router_state
struct dfly_router_sample ross_rsample;
};
/* had to pull some of the ROSS model stats collection stuff up here */
/* ROSS model instrumentation */
void custom_dragonfly_event_collect(terminal_custom_message *m, tw_lp *lp, char *buffer, int *collect_flag);
void custom_dragonfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer);
void custom_dfly_router_model_stat_collect(router_state *s, tw_lp *lp, char *buffer);
......@@ -1545,6 +1545,7 @@ static void packet_send(terminal_state * s, tw_bf * bf, terminal_custom_message
s->busy_time += (tw_now(lp) - s->last_buf_full[0]);
s->busy_time_sample += (tw_now(lp) - s->last_buf_full[0]);
s->ross_sample.busy_time_sample += (tw_now(lp) - s->last_buf_full[0]);
msg->saved_busy_time_ross = s->busy_time_ross_sample;
s->busy_time_ross_sample += (tw_now(lp) - s->last_buf_full[0]);
s->last_buf_full[0] = 0.0;
}
......@@ -3635,7 +3636,8 @@ tw_lptype dragonfly_custom_lps[] =
};
}
/* For ROSS event tracing */
/* ROSS Instrumentation layer */
// event tracing callback - used router and terminal LPs
void custom_dragonfly_event_collect(terminal_custom_message *m, tw_lp *lp, char *buffer, int *collect_flag)
{
(void)lp;
......@@ -3645,6 +3647,7 @@ void custom_dragonfly_event_collect(terminal_custom_message *m, tw_lp *lp, char
memcpy(buffer, &type, sizeof(type));
}
// GVT-based and real time sampling callback for terminals
void custom_dragonfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer)
{
(void)lp;
......@@ -3686,6 +3689,7 @@ void custom_dragonfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buf
return;
}
// GVT-based and real time sampling callback for routers
void custom_dfly_router_model_stat_collect(router_state *s, tw_lp *lp, char *buffer)
{
(void)lp;
......@@ -3735,7 +3739,7 @@ static void custom_router_register_model_types(st_model_types *base_type)
{
st_model_type_register(LP_CONFIG_NM_ROUT, base_type);
}
/*** END of ROSS event tracing additions */
/*** END of ROSS Instrumentation support */
/* returns the dragonfly lp type for lp registration */
static const tw_lptype* dragonfly_custom_get_cn_lp_type(void)
......
......@@ -338,6 +338,14 @@ struct terminal_state
/* for logging forward and reverse events */
long fwd_events;
long rev_events;
// for ROSS Instrumentation
long fin_chunks_ross_sample;
long data_size_ross_sample;
long fin_hops_ross_sample;
tw_stime fin_chunks_time_ross_sample;
tw_stime busy_time_ross_sample;
struct dfly_cn_sample ross_sample;
};
/* terminal event type (1-4) */
......@@ -479,8 +487,41 @@ struct router_state
long fwd_events;
long rev_events;
// for ROSS instrumentation
tw_stime* busy_time_ross_sample;
int64_t * link_traffic_ross_sample;
struct dfly_router_sample ross_rsample;
};
/* ROSS model instrumentation */
void dfly_plus_event_collect(terminal_plus_message *m, tw_lp *lp, char *buffer, int *collect_flag);
void dfly_plus_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer);
void dfly_plus_router_model_stat_collect(router_state *s, tw_lp *lp, char *buffer);
static void ross_dfly_plus_rsample_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct dfly_router_sample *sample);
static void ross_dfly_plus_rsample_rc_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct dfly_router_sample *sample);
static void ross_dfly_plus_sample_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct dfly_cn_sample *sample);
static void ross_dfly_plus_sample_rc_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct dfly_cn_sample *sample);
st_model_types dfly_plus_model_types[] = {
{(ev_trace_f) dfly_plus_event_collect,
sizeof(int),
(model_stat_f) dfly_plus_model_stat_collect,
sizeof(tw_lpid) + sizeof(long) * 2 + sizeof(double) + sizeof(tw_stime) *2,
(sample_event_f) ross_dfly_plus_sample_fn,
(sample_revent_f) ross_dfly_plus_sample_rc_fn,
sizeof(struct dfly_cn_sample) } ,
{(ev_trace_f) dfly_plus_event_collect,
sizeof(int),
(model_stat_f) dfly_plus_router_model_stat_collect,
0, //updated in router_plus_setup() since it's based on the radix
(sample_event_f) ross_dfly_plus_rsample_fn,
(sample_revent_f) ross_dfly_plus_rsample_rc_fn,
0 } , //updated in router_plus_setup() since it's based on the radix
{NULL, 0, NULL, 0, NULL, NULL, 0}
};
/* End of ROSS model instrumentation */
int dragonfly_plus_get_assigned_router_id(int terminal_id, const dragonfly_plus_param *p);
static short routing = MINIMAL;
......@@ -1258,6 +1299,16 @@ void router_plus_setup(router_state *r, tw_lp *lp)
r->busy_time = (tw_stime *) calloc(p->radix, sizeof(tw_stime));
r->busy_time_sample = (tw_stime *) calloc(p->radix, sizeof(tw_stime));
/* set up for ROSS stats sampling */
r->link_traffic_ross_sample = (int64_t*)calloc(p->radix, sizeof(int64_t));
r->busy_time_ross_sample = (tw_stime*)calloc(p->radix, sizeof(tw_stime));
if (g_st_model_stats)
lp->model_types->mstat_sz = sizeof(tw_lpid) + (sizeof(int64_t) + sizeof(tw_stime)) * p->radix;
if (g_st_use_analysis_lps && g_st_model_stats)
lp->model_types->sample_struct_sz = sizeof(struct dfly_router_sample) + (sizeof(tw_stime) + sizeof(int64_t)) * p->radix;
r->ross_rsample.busy_time = (tw_stime*)calloc(p->radix, sizeof(tw_stime));
r->ross_rsample.link_traffic_sample = (int64_t*)calloc(p->radix, sizeof(int64_t));
rc_stack_create(&r->st);
for (int i = 0; i < p->radix; i++) {
......@@ -1591,6 +1642,8 @@ static void packet_send_rc(terminal_state *s, tw_bf *bf, terminal_plus_message *
s->busy_time = msg->saved_total_time;
s->last_buf_full = msg->saved_busy_time;
s->busy_time_sample = msg->saved_sample_time;
s->ross_sample.busy_time_sample = msg->saved_sample_time;
s->busy_time_ross_sample = msg->saved_busy_time_ross;
}
}
return;
......@@ -1709,6 +1762,9 @@ static void packet_send(terminal_state *s, tw_bf *bf, terminal_plus_message *msg
s->busy_time += (tw_now(lp) - s->last_buf_full);
s->busy_time_sample += (tw_now(lp) - s->last_buf_full);
s->ross_sample.busy_time_sample += (tw_now(lp) - s->last_buf_full);
msg->saved_busy_time_ross = s->busy_time_ross_sample;
s->busy_time_ross_sample += (tw_now(lp) - s->last_buf_full);
s->last_buf_full = 0.0;
}
}
......@@ -1732,12 +1788,18 @@ static void packet_arrive_rc(terminal_state *s, tw_bf *bf, terminal_plus_message
N_finished_chunks--;
s->finished_chunks--;
s->fin_chunks_sample--;
s->ross_sample.fin_chunks_sample--;
s->fin_chunks_ross_sample--;
total_hops -= msg->my_N_hop;
s->total_hops -= msg->my_N_hop;
s->fin_hops_sample -= msg->my_N_hop;
s->ross_sample.fin_hops_sample -= msg->my_N_hop;
s->fin_hops_ross_sample -= msg->my_N_hop;
dragonfly_total_time = msg->saved_total_time;
s->fin_chunks_time = msg->saved_sample_time;
s->ross_sample.fin_chunks_time = msg->saved_sample_time;
s->fin_chunks_time_ross_sample = msg->saved_fin_chunks_ross;
s->total_time = msg->saved_avg_time;
struct qhash_head *hash_link = NULL;
......@@ -1776,6 +1838,8 @@ static void packet_arrive_rc(terminal_state *s, tw_bf *bf, terminal_plus_message
total_msg_sz -= msg->total_size;
s->total_msg_size -= msg->total_size;
s->data_size_sample -= msg->total_size;
s->ross_sample.data_size_sample -= msg->total_size;
s->data_size_ross_sample -= msg->total_size;
struct dfly_qhash_entry *d_entry_pop = (dfly_qhash_entry *) rc_stack_pop(s->st);
qhash_add(s->rank_tbl, &key, &(d_entry_pop->hash_link));
......@@ -1898,6 +1962,8 @@ static void packet_arrive(terminal_state *s, tw_bf *bf, terminal_plus_message *m
s->finished_chunks++;
/* Finished chunks per sample */
s->fin_chunks_sample++;
s->ross_sample.fin_chunks_sample++;
s->fin_chunks_ross_sample++;
/* WE do not allow self messages through dragonfly */
assert(lp->gid != msg->src_terminal_id);
......@@ -1927,6 +1993,9 @@ static void packet_arrive(terminal_state *s, tw_bf *bf, terminal_plus_message *m
/* save the sample time */
msg->saved_sample_time = s->fin_chunks_time;
s->fin_chunks_time += (tw_now(lp) - msg->travel_start_time);
s->ross_sample.fin_chunks_time += (tw_now(lp) - msg->travel_start_time);
msg->saved_fin_chunks_ross = s->fin_chunks_time_ross_sample;
s->fin_chunks_time_ross_sample += (tw_now(lp) - msg->travel_start_time);
/* save the total time per LP */
msg->saved_avg_time = s->total_time;
......@@ -1937,6 +2006,8 @@ static void packet_arrive(terminal_state *s, tw_bf *bf, terminal_plus_message *m
total_hops += msg->my_N_hop;
s->total_hops += msg->my_N_hop;
s->fin_hops_sample += msg->my_N_hop;
s->ross_sample.fin_hops_sample += msg->my_N_hop;
s->fin_hops_ross_sample += msg->my_N_hop;
mn_stats *stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
msg->saved_rcv_time = stat->recv_time;
......@@ -2015,6 +2086,8 @@ static void packet_arrive(terminal_state *s, tw_bf *bf, terminal_plus_message *m
bf->c7 = 1;
s->data_size_sample += msg->total_size;
s->ross_sample.data_size_sample += msg->total_size;
s->data_size_ross_sample += msg->total_size;
N_finished_msgs++;
total_msg_sz += msg->total_size;
s->total_msg_size += msg->total_size;
......@@ -3386,6 +3459,8 @@ static void router_packet_send_rc(router_state *s, tw_bf *bf, terminal_plus_mess
{
s->busy_time[output_port] = msg->saved_rcv_time;
s->busy_time_sample[output_port] = msg->saved_sample_time;
s->ross_rsample.busy_time[output_port] = msg->saved_sample_time;
s->busy_time_ross_sample[output_port] = msg->saved_busy_time_ross;
s->last_buf_full[output_port] = msg->saved_busy_time;
}
......@@ -3397,10 +3472,14 @@ static void router_packet_send_rc(router_state *s, tw_bf *bf, terminal_plus_mess
if (bf->c11) {
s->link_traffic[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size;
s->link_traffic_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size;
s->ross_rsample.link_traffic_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size;
s->link_traffic_ross_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size;
}
if (bf->c12) {
s->link_traffic[output_port] -= s->params->chunk_size;