Commit 113307ec authored by Misbah Mubarak's avatar Misbah Mubarak

Merging cortex with master

parents 9897e59c d5cf9422
......@@ -60,7 +60,6 @@ endif
if USE_DUMPI
AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
LDADD += ${DUMPI_LIBS}
if USE_CORTEX
if USE_PYTHON
if USE_CORTEX_PYTHON
......@@ -73,5 +72,6 @@ endif
AM_CPPFLAGS += ${CORTEX_CFLAGS} -DENABLE_CORTEX=1
LDADD += ${CORTEX_LIBS}
endif
LDADD += ${DUMPI_LIBS}
endif
/*
* Copyright (C) 2017 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef CODES_MPI_REPLAY_H
#define CODES_MPI_REPLAY_H
#ifdef __cplusplus
extern "C" {
#endif
#include <mpi.h>
int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv );
#ifdef __cplusplus
}
#endif
#endif /* CODES_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
......@@ -14,6 +14,9 @@ extern "C" {
#include <ross.h>
#include <assert.h>
#include <mpi.h>
extern MPI_Comm MPI_COMM_CODES;
// for printf conversions: shorthand for cast to long long unsigned format (llu)
#define LLU(x) ((unsigned long long)(x))
......
......@@ -18,6 +18,11 @@ extern "C" {
#include <codes/codes-mapping-context.h>
#include <stdint.h>
#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif
#define PULL_MSG_SIZE 128
#define MAX_NAME_LENGTH 256
......@@ -406,6 +411,11 @@ void model_net_print_stats(tw_lpid lpid, mn_stats mn_stats_array[]);
/* find model-net statistics */
mn_stats* model_net_find_stats(char const * category, mn_stats mn_stats_array[]);
#ifdef ENABLE_CORTEX
/* structure that gives access to the topology functions */
extern cortex_topology model_net_topology;
#endif
#ifdef __cplusplus
}
#endif
......
lib_LTLIBRARIES += src/libcodes.la
lib_LTLIBRARIES += src/libcodes.la src/libcodes_mpi_replay.la
BUILT_SOURCES += src/modelconfig/configparser.h \
src/modelconfig/configlex.h
......@@ -95,6 +95,7 @@ nobase_include_HEADERS = \
codes/net/simplenet-upd.h \
codes/net/simplep2p.h \
codes/net/torus.h \
codes/codes-mpi-replay.h \
codes/configfile.h
......@@ -139,6 +140,7 @@ src_libcodes_la_SOURCES = \
src/util/jobmap-impl/jobmap-list.c\
src/util/jobmap-impl/jobmap-identity.c\
src/util/codes-mapping-context.c \
src/util/codes-comm.c \
src/workload/codes-workload.c \
src/workload/methods/codes-iolang-wrkld.c \
src/workload/methods/codes-checkpoint-wrkld.c \
......@@ -160,6 +162,8 @@ src_libcodes_la_SOURCES = \
src/networks/model-net/model-net-sched-impl.h \
src/networks/model-net/model-net-sched-impl.c
src_libcodes_mpi_replay_la_SOURCES = \
src/network-workloads/model-net-mpi-replay.c
#codes/codes-nw-workload.h
#src/network-workload/codes-nw-workload.c
......@@ -175,6 +179,7 @@ src_util_templates_lp_template_dummy_main_SOURCES = \
# get rid of annoying unused function in template
bin_PROGRAMS += src/workload/codes-workload-dump
bin_PROGRAMS += src/networks/model-net/topology-test
bin_PROGRAMS += src/network-workloads/model-net-mpi-replay
bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
bin_PROGRAMS += src/network-workloads/model-net-synthetic
......@@ -187,10 +192,13 @@ src_workload_codes_workload_dump_SOURCES = \
src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c
src_network_workloads_model_net_synthetic_slimfly_SOURCES = src/network-workloads/model-net-synthetic-slimfly.c
src_network_workloads_model_net_mpi_replay_SOURCES = src/network-workloads/model-net-mpi-replay.c
src_network_workloads_model_net_mpi_replay_SOURCES = \
src/network-workloads/model-net-mpi-replay.c \
src/network-workloads/model-net-mpi-replay-main.c
src_network_workloads_model_net_mpi_replay_CFLAGS = $(AM_CFLAGS)
src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c
src_network_workloads_model_net_synthetic_custom_dfly_SOURCES = src/network-workloads/model-net-synthetic-custom-dfly.c
src_networks_model_net_topology_test_SOURCES = src/networks/model-net/topology-test.c
#bin_PROGRAMS += src/network-workload/codes-nw-test
......
/*
* Copyright (C) 2014 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include <mpi.h>
#include "codes/codes-mpi-replay.h"
int main(int argc, char** argv) {
MPI_Init(&argc,&argv);
// int rank, size;
// MPI_Comm_rank(MPI_COMM_WORLD,&rank);
// MPI_Comm_size(MPI_COMM_WORLD,&size);
// MPI_Comm comm;
// MPI_Comm_split(MPI_COMM_WORLD, rank < 2, rank, &comm);
// if(rank < 2)
// modelnet_mpi_replay(comm,&argc,&argv);
modelnet_mpi_replay(MPI_COMM_WORLD,&argc,&argv);
int flag;
MPI_Finalized(&flag);
if(!flag) MPI_Finalize();
return 0;
}
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
......@@ -2209,17 +2209,21 @@ static int msg_size_hash_compare(
return 0;
}
int main( int argc, char** argv )
int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
{
int rank, nprocs;
int num_nets;
int* net_ids;
MPI_COMM_CODES = comm;
tw_comm_set(MPI_COMM_CODES);
g_tw_ts_end = s_to_ns(60*5); /* five minutes, in nsecs */
workload_type[0]='\0';
tw_opt_add(app_opt);
tw_init(&argc, &argv);
tw_init(argc, argv);
if(strcmp(workload_type, "dumpi") != 0)
{
......@@ -2231,6 +2235,7 @@ int main( int argc, char** argv )
#ifdef ENABLE_CORTEX_PYTHON
" --cortex-file=cortex-file-name"
" --cortex-class=cortex-class-name"
" --cortex-gen=cortex-function-name"
#endif
" -- config-file-name\n"
"See model-net/doc/README.dragonfly.txt and model-net/doc/README.torus.txt"
......@@ -2239,6 +2244,8 @@ int main( int argc, char** argv )
return -1;
}
jobmap_ctx = NULL; // make sure it's NULL if it's not used
if(strlen(workloads_conf_file) > 0)
{
FILE *name_file = fopen(workloads_conf_file, "r");
......@@ -2278,11 +2285,16 @@ int main( int argc, char** argv )
strcpy(file_name_of_job[0], workload_file);
num_traces_of_job[0] = num_net_traces;
alloc_spec = 0;
if(strlen(alloc_file) > 0) {
alloc_spec = 1;
jobmap_p.alloc_file = alloc_file;
jobmap_ctx = codes_jobmap_configure(CODES_JOBMAP_LIST, &jobmap_p);
}
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load((*argv)[2], MPI_COMM_CODES, &config);
nw_add_lp_type();
model_net_register();
......@@ -2341,7 +2353,7 @@ int main( int argc, char** argv )
do_lp_io = 1;
/* initialize lp io */
int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_prepare failure");
}
tw_run();
......@@ -2359,19 +2371,19 @@ int main( int argc, char** argv )
double total_avg_wait_time, total_max_wait_time;
double total_avg_recv_time, total_max_recv_time;
MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&max_comm_time, &max_comm_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&max_time, &max_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&max_comm_time, &max_comm_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&max_time, &max_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&max_wait_time, &total_max_wait_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&max_send_time, &total_max_send_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&max_recv_time, &total_max_recv_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&max_wait_time, &total_max_wait_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&max_send_time, &total_max_send_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&max_recv_time, &total_max_recv_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
assert(num_net_traces);
......@@ -2385,7 +2397,7 @@ int main( int argc, char** argv )
total_max_recv_time, total_avg_recv_time/num_net_traces,
total_max_wait_time, total_avg_wait_time/num_net_traces);
if (do_lp_io){
int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
......
......@@ -358,10 +358,10 @@ int main(
return 0;
}
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
MPI_Comm_size(MPI_COMM_CODES, &nprocs);
configuration_load(argv[2], MPI_COMM_WORLD, &config);
configuration_load(argv[2], MPI_COMM_CODES, &config);
model_net_register();
svr_add_lp_type();
......@@ -401,12 +401,12 @@ int main(
{
do_lp_io = 1;
int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_prepare failure");
}
tw_run();
if (do_lp_io){
int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
assert(ret == 0 || !"lp_io_flush failure");
}
model_net_report_stats(net_id);
......
This diff is collapsed.
......@@ -22,6 +22,11 @@
#include "codes/quickhash.h"
#include "codes/rc-stack.h"
#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif
#define CREDIT_SZ 8
#define MEAN_PROCESS 1.0
......@@ -49,6 +54,11 @@
#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[DRAGONFLY_ROUTER])
#define LP_METHOD_NM_ROUT (model_net_method_names[DRAGONFLY_ROUTER])
#ifdef ENABLE_CORTEX
/* This structure is defined at the end of the file */
extern cortex_topology dragonfly_cortex_topology;
#endif
int debug_slot_count = 0;
long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount;
long packet_gen = 0, packet_fin = 0;
......@@ -573,7 +583,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
p->total_routers = p->num_groups * p->num_routers;
p->total_terminals = p->total_routers * p->num_cn;
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
if(!rank) {
printf("\n Total nodes %d routers %d groups %d radix %d \n",
p->num_cn * p->total_routers, p->total_routers, p->num_groups,
......@@ -599,6 +609,9 @@ static void dragonfly_configure(){
if (anno_map->has_unanno_lp > 0){
dragonfly_read_config(NULL, &all_params[anno_map->num_annos]);
}
#ifdef ENABLE_CORTEX
model_net_topology = dragonfly_cortex_topology;
#endif
}
/* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */
......@@ -610,20 +623,20 @@ static void dragonfly_report_stats()
int total_minimal_packets, total_nonmin_packets;
long total_gen, total_fin;
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
{
MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
}
/* print statistics */
......@@ -3355,3 +3368,75 @@ struct model_net_method dragonfly_router_method =
.mn_trace_register = router_register_trace,
.mn_get_trace_type = dragonfly_get_trace_types,
};
#ifdef ENABLE_CORTEX
static int dragonfly_get_number_of_compute_nodes(void* topo) {
// TODO
return -1;
}
static int dragonfly_get_number_of_routers(void* topo) {
// TODO
return -1;
}
static double dragonfly_get_router_link_bandwidth(void* topo, router_id_t r1, router_id_t r2) {
// TODO
return -1.0;
}
static double dragonfly_get_compute_node_bandwidth(void* topo, cn_id_t node) {
// TODO
return -1.0;
}
static int dragonfly_get_router_neighbor_count(void* topo, router_id_t r) {
// TODO
return 0;
}
static void dragonfly_get_router_neighbor_list(void* topo, router_id_t r, router_id_t* neighbors) {
// TODO
}
static int dragonfly_get_router_location(void* topo, router_id_t r, int32_t* location, int size) {
// TODO
return 0;
}
static int dragonfly_get_compute_node_location(void* topo, cn_id_t node, int32_t* location, int size) {
// TODO
return 0;
}
static router_id_t dragonfly_get_router_from_compute_node(void* topo, cn_id_t node) {
// TODO
return -1;
}
static int dragonfly_get_router_compute_node_count(void* topo, router_id_t r) {
// TODO
return 0;
}
static void dragonfly_get_router_compute_node_list(void* topo, router_id_t r, cn_id_t* nodes) {
// TODO
}
cortex_topology dragonfly_cortex_topology = {
.internal = NULL,
.get_number_of_routers = dragonfly_get_number_of_routers,
.get_number_of_compute_nodes = dragonfly_get_number_of_compute_nodes,
.get_router_link_bandwidth = dragonfly_get_router_link_bandwidth,
.get_compute_node_bandwidth = dragonfly_get_compute_node_bandwidth,
.get_router_neighbor_count = dragonfly_get_router_neighbor_count,
.get_router_neighbor_list = dragonfly_get_router_neighbor_list,
.get_router_location = dragonfly_get_router_location,
.get_compute_node_location = dragonfly_get_compute_node_location,
.get_router_from_compute_node = dragonfly_get_router_from_compute_node,
.get_router_compute_node_count = dragonfly_get_router_compute_node_count,
.get_router_compute_node_list = dragonfly_get_router_compute_node_list,
};
#endif
......@@ -13,6 +13,11 @@
#include <ctype.h>
#include <search.h>
#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif
#define CREDIT_SIZE 8
#define MEAN_PROCESS 1.0
......@@ -34,6 +39,11 @@
#define LP_CONFIG_NM (model_net_lp_config_names[FATTREE])
#define LP_METHOD_NM (model_net_method_names[FATTREE])
#ifdef ENABLE_CORTEX
/* This structure is defined at the end of the file */
extern cortex_topology fattree_cortex_topology;
#endif
#if DEBUG_RC
//Reverse Compute Debug Variables
long long packet_event_f = 0;
......@@ -546,7 +556,7 @@ static void dot_write_open_file(FILE **fout)
char *dot_file_prefix = dot_file_p;
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
sprintf(file_name, "%s/%s.dot.%d", dir_name, dot_file_prefix, rank);
*fout = fopen(file_name, "w+");
if(*fout == NULL) {
......@@ -891,6 +901,9 @@ static void fattree_configure(){
if (anno_map->has_unanno_lp > 0){
fattree_read_config(NULL, &all_params[anno_map->num_annos]);
}
#ifdef ENABLE_CORTEX
model_net_topology = fattree_cortex_topology;
#endif
}
/* initialize a fattree compute node terminal */
......@@ -1348,22 +1361,22 @@ static void fattree_report_stats()
long long ts_buffer_f = 0;
long long ts_buffer_r = 0;
MPI_Reduce( &packet_event_f, &t_packet_event_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_event_r, &t_packet_event_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_generate_f, &tt_generate_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_generate_r, &tt_generate_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_send_f, &tt_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_send_r, &tt_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_arrive_f, &tt_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_arrive_r, &tt_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_buffer_f, &tt_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &t_buffer_r, &tt_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &s_send_f, &ts_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &s_send_r, &ts_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &s_arrive_f, &ts_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &s_arrive_r, &ts_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &s_buffer_f, &ts_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &s_buffer_r, &ts_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &packet_event_f, &t_packet_event_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &packet_event_r, &t_packet_event_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_generate_f, &tt_generate_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_generate_r, &tt_generate_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_send_f, &tt_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_send_r, &tt_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_arrive_f, &tt_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_arrive_r, &tt_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_buffer_f, &tt_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &t_buffer_r, &tt_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &s_send_f, &ts_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &s_send_r, &ts_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &s_arrive_f, &ts_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &s_arrive_r, &ts_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &s_buffer_f, &ts_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &s_buffer_r, &ts_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
if(!g_tw_mynode)
{
......@@ -1399,16 +1412,16 @@ static void fattree_report_stats()
tw_stime avg_time, max_time;
long total_gen, total_fin;
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &fattree_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &fattree_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &fattree_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &fattree_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
MPI_Reduce( &fattree_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &fattree_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce( &fattree_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
MPI_Reduce( &fattree_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
/* print statistics */
if(!g_tw_mynode)
......@@ -2836,7 +2849,7 @@ void fattree_switch_final(switch_state * s, tw_lp * lp)
// char *stats_file = getenv("TRACER_LINK_FILE");
// if(stats_file != NULL) {
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
char file_name[512];
sprintf(file_name, "%s.%d", "tracer_stats_file", rank);
FILE *fout = fopen(file_name, "a");
......@@ -2992,3 +3005,74 @@ struct model_net_method fattree_method =
.mn_collective_call_rc = NULL
};
#ifdef ENABLE_CORTEX
static int fattree_get_number_of_compute_nodes(void* topo) {
// TODO
return -1;
}
static int fattree_get_number_of_routers(void* topo) {
// TODO
return -1;
}
static double fattree_get_router_link_bandwidth(void* topo, router_id_t r1, router_id_t r2) {
// TODO
return -1.0;
}
static double fattree_get_compute_node_bandwidth(void* topo, cn_id_t node) {
// TODO
return -1.0;
}
static int fattree_get_router_neighbor_count(void* topo, router_id_t r) {
// TODO
return 0;
}
static void fattree_get_router_neighbor_list(void* topo, router_id_t r, router_id_t* neighbors) {
// TODO
}
static int fattree_get_router_location(void* topo, router_id_t r, int32_t* location, int size) {
// TODO
return 0;
}
static int fattree_get_compute_node_location(void* topo, cn_id_t node, int32_t* location, int size) {
// TODO
return 0;
}
static router_id_t fattree_get_router_from_compute_node(void* topo, cn_id_t node) {
// TODO
return -1;
}
static int fattree_get_router_compute_node_count(void* topo, router_id_t r) {
// TODO
return 0;
}
static void fattree_get_router_compute_node_list(void* topo, router_id_t r, cn_id_t* nodes) {
// TODO
}
cortex_topology fattree_cortex_topology = {
.internal = NULL,
.get_number_of_compute_nodes = fattree_get_number_of_compute_nodes,
.get_number_of_routers = fattree_get_number_of_routers,
.get_router_link_bandwidth = fattree_get_router_link_bandwidth,
.get_compute_node_bandwidth = fattree_get_compute_node_bandwidth,
.get_router_neighbor_count = fattree_get_router_neighbor_count,
.get_router_neighbor_list = fattree_get_router_neighbor_list,
.get_router_location = fattree_get_router_location,
.get_compute_node_location = fattree_get_compute_node_location,
.get_router_from_compute_node = fattree_get_router_from_compute_node,
.get_router_compute_node_count = fattree_get_router_compute_node_count,
.get_router_compute_node_list = fattree_get_router_compute_node_list,
};
#endif
......@@ -14,6 +14,7 @@
#include "codes/codes.h"
#include <codes/codes_mapping.h>
#define STR_SIZE 16
#define PROC_TIME 10.0
......@@ -637,6 +638,11 @@ tw_lpid model_net_find_local_device_mctx(
sender_gid);
}
#ifdef ENABLE_CORTEX
/* structure that gives access to the topology functions */
cortex_topology model_net_topology;
#endif
/*
* Local variables:
* c-indent-level: 4
......
......@@ -554,7 +554,7 @@ static void slimfly_read_config(const char * anno, slimfly_param *params){
slim_total_routers_noah = p->num_groups * p->num_routers;
slim_total_terminals_noah = p->slim_total_routers * p->num_cn;
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_rank(MPI_COMM_CODES, &rank);
if(!rank) {
printf("\n Total nodes %d total routers %d total groups %d num_terminals %d num_routers %d radix %d local_channels %d global_channels %d \n",
p->num_cn * p->slim_total_routers, p->slim_total_routers, p->num_groups, p->num_cn, p->num_routers,
......
/* turning on track lp will generate a lot of output messages */
#include <ross.h>
#include <inttypes.h>
#include "codes/codes-workload.h"
#include "codes/codes.h"
#ifdef ENABLE_CORTEX
#include <cortex/topology.h>
#endif
#include "codes/model-net.h"
/* TODO: Replace NUM_NODES and num_routers once we have functions to fetch
* topology node and router counts. Right now it is hard coded for Theta. */
extern struct cortex_topology dragonfly_custom_cortex_topology;
int main(int argc, char** argv) {
#ifndef ENABLE_CORTEX
printf("\n Cortex needs to be enabled in order to run the test. See <https://xgitlab.cels.anl.gov/codes/codes/wikis/codes-cortex-install> ");
return -1;
#else
if(argc < 2)
{