Skip to content

GitLab

Commit 25f66fdb authored by Caitlin Ross's avatar Caitlin Ross
Browse files

changing MPI_COMM_WORLD to MPI_COMM_CODES and other support for using ROSS-Damaris

parent 9d87ca60
Hide whitespace changes
Inline Side-by-side
Showing with 128 additions and 69 deletions
codes/codes.h
View file @ 25f66fdb
......@@ -97,6 +97,8 @@ static inline void codes_local_latency_reverse(tw_lp *lp)
return;
}
void codes_comm_update();
#ifdef __cplusplus
}
#endif
......
src/network-workloads/conf/modelnet-mpi-test-dfly-amg-1728.conf
View file @ 25f66fdb
......@@ -23,6 +23,6 @@ PARAMS
local_bandwidth="5.25";
global_bandwidth="4.7";
cn_bandwidth="5.25";
message_size="592";
message_size="608";
routing="adaptive";
}
src/network-workloads/model-net-dumpi-traces-dump.c
View file @ 25f66fdb
......@@ -414,6 +414,11 @@ int main( int argc, char** argv )
workload_type[0]='\0';
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(strlen(workload_file) == 0)
{
......@@ -423,10 +428,10 @@ int main( int argc, char** argv )
return -1;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
nw_add_lp_type();
......@@ -446,20 +451,20 @@ int main( int argc, char** argv )
double total_avg_comp_time;
long overall_sends, overall_recvs, overall_waits, overall_cols;
MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_col_time, &avg_col_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_compute_time, &total_avg_comp_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_sends, &overall_sends, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_recvs, &overall_recvs, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_waits, &overall_waits, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_collectives, &overall_cols, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_col_time, &avg_col_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_compute_time, &total_avg_comp_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_sends, &overall_sends, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_recvs, &overall_recvs, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_waits, &overall_waits, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_collectives, &overall_cols, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
if(!g_tw_mynode)
printf("\n Total bytes sent %lld recvd %lld \n avg runtime %lf \n avg comm time %lf avg compute time %lf \n avg send time %lf \n avg recv time %lf \n avg wait time %lf \n total sends %ld total recvs %ld total waits %ld total collectives %ld ", total_bytes_sent, total_bytes_recvd,
......@@ -470,6 +475,9 @@ int main( int argc, char** argv )
total_avg_recv_time/num_net_lps,
total_avg_wait_time/num_net_lps,
overall_sends, overall_recvs, overall_waits, overall_cols);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
......
src/network-workloads/model-net-mpi-replay.c
View file @ 25f66fdb
......@@ -2626,6 +2626,11 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
workload_type[0]='\0';
tw_opt_add(app_opt);
tw_init(argc, argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(strcmp(workload_type, "dumpi") != 0)
{
......@@ -2793,7 +2798,7 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
MPI_Reduce(&max_recv_time, &total_max_recv_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&total_syn_data, &g_total_syn_data, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&total_syn_data, &g_total_syn_data, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
assert(num_net_traces);
......@@ -2822,6 +2827,9 @@ int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
if(alloc_spec)
codes_jobmap_destroy(jobmap_ctx);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
......
src/network-workloads/model-net-synthetic-custom-dfly.c
View file @ 25f66fdb
......@@ -407,6 +407,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -470,6 +475,9 @@ int main(
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
}
......
src/network-workloads/model-net-synthetic-fattree.c
View file @ 25f66fdb
......@@ -436,6 +436,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
offset = 1;
......@@ -446,10 +451,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
......@@ -484,7 +489,7 @@ int main(
printf("num_nodes:%d \n",num_nodes);
if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0)
if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_CODES) < 0)
{
return(-1);
}
......@@ -516,11 +521,13 @@ int main(
}
#endif
if(lp_io_flush(handle, MPI_COMM_WORLD) < 0)
if(lp_io_flush(handle, MPI_COMM_CODES) < 0)
{
return(-1);
}
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
#if PARAMS_LOG
......
src/network-workloads/model-net-synthetic-slimfly.c
View file @ 25f66fdb
......@@ -494,6 +494,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -502,10 +507,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
svr_add_lp_type();
......@@ -526,7 +531,7 @@ int main(
num_nodes_per_grp = num_routers_per_grp * num_servers_per_rep;
total_routers = num_routers_per_grp * num_routers_per_grp * 2;
/* if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0)
/* if(lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_CODES) < 0)
{
return(-1);
}
......@@ -536,7 +541,7 @@ int main(
{
do_lp_io = 1;
int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_prepare failure");
}
......@@ -558,7 +563,7 @@ int main(
if (do_lp_io){
int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_flush failure");
}
......@@ -578,12 +583,15 @@ int main(
#endif
}
/* if(lp_io_flush(handle, MPI_COMM_WORLD) < 0)
/* if(lp_io_flush(handle, MPI_COMM_CODES) < 0)
{
assert(ret == 0 || !"lp_io_flush failure");
return(-1);
}
*/
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
if(rank == 0)
......
src/network-workloads/model-net-synthetic.c
View file @ 25f66fdb
......@@ -407,6 +407,11 @@ int main(
tw_opt_add(app_opt);
tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
if(g_st_ross_rank)
{ // keep damaris ranks from running code between here up until tw_end()
#endif
codes_comm_update();
if(argc < 2)
{
......@@ -415,10 +420,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
svr_add_lp_type();
......@@ -457,15 +462,18 @@ int main(
{
do_lp_io = 1;
int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_prepare failure");
}
tw_run();
if (do_lp_io){
int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
#ifdef USE_RDAMARIS
} // end if(g_st_ross_rank)
#endif
tw_end();
return 0;
}
......
src/networks/model-net/express-mesh.C
View file @ 25f66fdb
......@@ -435,7 +435,7 @@ static void local_read_config(const char * anno, local_param *params){
p->total_terminals = p->total_routers * p->num_cn;
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
if(!rank) {
printf("\n Total nodes %d routers %d radix %d \n",
p->total_terminals, p->total_routers, p->radix);
......@@ -480,22 +480,22 @@ static void local_report_stats()
long total_gen, total_fin;
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG,
MPI_SUM, 0, MPI_COMM_WORLD);
MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM,
0, MPI_COMM_WORLD);
0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG,
MPI_SUM, 0, MPI_COMM_WORLD);
MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &local_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &local_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
/* print statistics */
if(!g_tw_mynode)
......
src/networks/model-net/net-template.C
View file @ 25f66fdb
......@@ -401,22 +401,22 @@ static void local_report_stats()
long total_gen, total_fin;
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG,
MPI_SUM, 0, MPI_COMM_WORLD);
MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM,
0, MPI_COMM_WORLD);
0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG,
MPI_SUM, 0, MPI_COMM_WORLD);
MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &local_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &local_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0,
MPI_COMM_WORLD);
MPI_COMM_CODES);
MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
/* print statistics */
if(!g_tw_mynode)
......
src/networks/model-net/slimfly.c
View file @ 25f66fdb
......@@ -638,19 +638,19 @@ static void slimfly_report_stats()
float throughput_avg = 0.0;
float throughput_avg2 = 0.0;
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &slimfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &slimfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&pe_throughput_percent, &throughput_avg, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&pe_throughput, &throughput_avg2, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &slimfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &slimfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&pe_throughput_percent, &throughput_avg, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&pe_throughput, &throughput_avg2, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
/* print statistics */
if(!g_tw_mynode)
......
src/util/codes-comm.c
View file @ 25f66fdb
......@@ -5,9 +5,19 @@
*/
#include <mpi.h>
#include <ross.h>
MPI_Comm MPI_COMM_CODES = MPI_COMM_WORLD;
/*
* Needs to be called AFTER tw_init() because in tw_init,
* ROSS may split the MPI_COMM_ROSS communicator
*/
void codes_comm_update()
{
MPI_COMM_CODES = MPI_COMM_ROSS;
}
/*
* Local variables:
* c-indent-level: 4
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment