Merging cortex with master

Makefile.am

@@ -60,7 +60,6 @@ endif
 if USE_DUMPI
 AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
 src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
-LDADD += ${DUMPI_LIBS}
 if USE_CORTEX
 if USE_PYTHON
 if USE_CORTEX_PYTHON
@@ -73,5 +72,6 @@ endif
 AM_CPPFLAGS += ${CORTEX_CFLAGS} -DENABLE_CORTEX=1
 LDADD += ${CORTEX_LIBS}
 endif
+LDADD += ${DUMPI_LIBS}
 endif
 

codes/codes-mpi-replay.h

+/*
+ * Copyright (C) 2017 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+
+#ifndef CODES_MPI_REPLAY_H
+#define CODES_MPI_REPLAY_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <mpi.h>
+
+int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CODES_H */
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

codes/codes.h

@@ -14,6 +14,9 @@ extern "C" {
 
 #include <ross.h>
 #include <assert.h>
+#include <mpi.h>
+
+extern MPI_Comm MPI_COMM_CODES;
 
 // for printf conversions: shorthand for cast to long long unsigned format (llu)
 #define LLU(x) ((unsigned long long)(x))

codes/model-net.h

@@ -18,6 +18,11 @@ extern "C" {
 #include <codes/codes-mapping-context.h>
 #include <stdint.h>
 
+#ifdef ENABLE_CORTEX
+#include <cortex/cortex.h>
+#include <cortex/topology.h>
+#endif
+
 #define PULL_MSG_SIZE 128
 
 #define MAX_NAME_LENGTH 256
@@ -406,6 +411,11 @@ void model_net_print_stats(tw_lpid lpid, mn_stats mn_stats_array[]);
 /* find model-net statistics */
 mn_stats* model_net_find_stats(char const * category, mn_stats mn_stats_array[]);
 
+#ifdef ENABLE_CORTEX
+/* structure that gives access to the topology functions */
+extern cortex_topology model_net_topology;
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/Makefile.subdir

-lib_LTLIBRARIES += src/libcodes.la
+lib_LTLIBRARIES += src/libcodes.la src/libcodes_mpi_replay.la
 
 BUILT_SOURCES += src/modelconfig/configparser.h \
                  src/modelconfig/configlex.h
@@ -95,6 +95,7 @@ nobase_include_HEADERS = \
 	codes/net/simplenet-upd.h \
 	codes/net/simplep2p.h \
 	codes/net/torus.h \
+  codes/codes-mpi-replay.h \
 	codes/configfile.h
 
 
@@ -139,6 +140,7 @@ src_libcodes_la_SOURCES =  \
 	src/util/jobmap-impl/jobmap-list.c\
 	src/util/jobmap-impl/jobmap-identity.c\
 	src/util/codes-mapping-context.c \
+  src/util/codes-comm.c \
     src/workload/codes-workload.c \
     src/workload/methods/codes-iolang-wrkld.c \
     src/workload/methods/codes-checkpoint-wrkld.c \
@@ -160,6 +162,8 @@ src_libcodes_la_SOURCES =  \
 	src/networks/model-net/model-net-sched-impl.h \
 	src/networks/model-net/model-net-sched-impl.c 
 
+src_libcodes_mpi_replay_la_SOURCES = \
+  src/network-workloads/model-net-mpi-replay.c
 
 #codes/codes-nw-workload.h
 #src/network-workload/codes-nw-workload.c
@@ -175,6 +179,7 @@ src_util_templates_lp_template_dummy_main_SOURCES = \
 # get rid of annoying unused function in template
 
 bin_PROGRAMS += src/workload/codes-workload-dump
+bin_PROGRAMS += src/networks/model-net/topology-test
 bin_PROGRAMS += src/network-workloads/model-net-mpi-replay
 bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
 bin_PROGRAMS += src/network-workloads/model-net-synthetic
@@ -187,10 +192,13 @@ src_workload_codes_workload_dump_SOURCES = \
 
 src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c 
 src_network_workloads_model_net_synthetic_slimfly_SOURCES = src/network-workloads/model-net-synthetic-slimfly.c
-src_network_workloads_model_net_mpi_replay_SOURCES = src/network-workloads/model-net-mpi-replay.c 
+src_network_workloads_model_net_mpi_replay_SOURCES = \
+	src/network-workloads/model-net-mpi-replay.c \
+  src/network-workloads/model-net-mpi-replay-main.c
+src_network_workloads_model_net_mpi_replay_CFLAGS = $(AM_CFLAGS)
 src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c 
 src_network_workloads_model_net_synthetic_custom_dfly_SOURCES = src/network-workloads/model-net-synthetic-custom-dfly.c 
-
+src_networks_model_net_topology_test_SOURCES = src/networks/model-net/topology-test.c
 
 #bin_PROGRAMS += src/network-workload/codes-nw-test
 

src/network-workloads/model-net-mpi-replay-main.c

+/*
+ * Copyright (C) 2014 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+#include <mpi.h>
+#include "codes/codes-mpi-replay.h"
+
+
+int main(int argc, char** argv) {
+  MPI_Init(&argc,&argv);
+  
+//  int rank, size;
+//  MPI_Comm_rank(MPI_COMM_WORLD,&rank);
+//  MPI_Comm_size(MPI_COMM_WORLD,&size);
+
+//  MPI_Comm comm;
+//  MPI_Comm_split(MPI_COMM_WORLD, rank < 2, rank, &comm);
+
+//  if(rank < 2)
+//  	modelnet_mpi_replay(comm,&argc,&argv);
+
+   modelnet_mpi_replay(MPI_COMM_WORLD,&argc,&argv);
+  int flag;
+  MPI_Finalized(&flag);
+	if(!flag) MPI_Finalize();
+	return 0;
+}
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

src/network-workloads/model-net-mpi-replay.c

@@ -2209,17 +2209,21 @@ static int msg_size_hash_compare(
 
     return 0;
 }
-int main( int argc, char** argv )
+int modelnet_mpi_replay(MPI_Comm comm, int* argc, char*** argv )
 {
   int rank, nprocs;
   int num_nets;
   int* net_ids;
 
+  MPI_COMM_CODES = comm;
+ 
+  tw_comm_set(MPI_COMM_CODES);
+
   g_tw_ts_end = s_to_ns(60*5); /* five minutes, in nsecs */
 
   workload_type[0]='\0';
   tw_opt_add(app_opt);
-  tw_init(&argc, &argv);
+  tw_init(argc, argv);
 
   if(strcmp(workload_type, "dumpi") != 0)
     {
@@ -2231,6 +2235,7 @@ int main( int argc, char** argv )
 #ifdef ENABLE_CORTEX_PYTHON
 		" --cortex-file=cortex-file-name"
 		" --cortex-class=cortex-class-name"
+		" --cortex-gen=cortex-function-name"
 #endif
 		" -- config-file-name\n"
                 "See model-net/doc/README.dragonfly.txt and model-net/doc/README.torus.txt"
@@ -2239,6 +2244,8 @@ int main( int argc, char** argv )
 	return -1;
     }
 
+	jobmap_ctx = NULL; // make sure it's NULL if it's not used
+
     if(strlen(workloads_conf_file) > 0)
     {
         FILE *name_file = fopen(workloads_conf_file, "r");
@@ -2278,11 +2285,16 @@ int main( int argc, char** argv )
         strcpy(file_name_of_job[0], workload_file);
         num_traces_of_job[0] = num_net_traces;
         alloc_spec = 0;
+		if(strlen(alloc_file) > 0) {
+			alloc_spec = 1;
+			jobmap_p.alloc_file = alloc_file;
+			jobmap_ctx = codes_jobmap_configure(CODES_JOBMAP_LIST, &jobmap_p);
+		}
     }
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+    MPI_Comm_rank(MPI_COMM_CODES, &rank);
+    MPI_Comm_size(MPI_COMM_CODES, &nprocs);
 
-   configuration_load(argv[2], MPI_COMM_WORLD, &config);
+   configuration_load((*argv)[2], MPI_COMM_CODES, &config);
 
    nw_add_lp_type();
    model_net_register();
@@ -2341,7 +2353,7 @@ int main( int argc, char** argv )
         do_lp_io = 1;
         /* initialize lp io */
         int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
-        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
+        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
         assert(ret == 0 || !"lp_io_prepare failure");
     }
    tw_run();
@@ -2359,19 +2371,19 @@ int main( int argc, char** argv )
      double total_avg_wait_time, total_max_wait_time;
      double total_avg_recv_time, total_max_recv_time;
 
-    MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-    MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&max_comm_time, &max_comm_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&max_time, &max_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    MPI_Reduce(&num_bytes_sent, &total_bytes_sent, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+    MPI_Reduce(&num_bytes_recvd, &total_bytes_recvd, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce(&max_comm_time, &max_comm_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
+   MPI_Reduce(&max_time, &max_run_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
+   MPI_Reduce(&avg_time, &avg_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
 
-   MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&max_wait_time, &total_max_wait_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&max_send_time, &total_max_send_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&max_recv_time, &total_max_recv_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+   MPI_Reduce(&avg_recv_time, &total_avg_recv_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce(&avg_comm_time, &avg_comm_run_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce(&max_wait_time, &total_max_wait_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
+   MPI_Reduce(&max_send_time, &total_max_send_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
+   MPI_Reduce(&max_recv_time, &total_max_recv_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
+   MPI_Reduce(&avg_wait_time, &total_avg_wait_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce(&avg_send_time, &total_avg_send_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
 
    assert(num_net_traces);
 
@@ -2385,7 +2397,7 @@ int main( int argc, char** argv )
 			total_max_recv_time, total_avg_recv_time/num_net_traces,
 			total_max_wait_time, total_avg_wait_time/num_net_traces);
     if (do_lp_io){
-        int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
+        int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
         assert(ret == 0 || !"lp_io_flush failure");
     }
    model_net_report_stats(net_id);

src/network-workloads/model-net-synthetic-custom-dfly.c

@@ -358,10 +358,10 @@ int main(
             return 0;
     }
 
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+    MPI_Comm_rank(MPI_COMM_CODES, &rank);
+    MPI_Comm_size(MPI_COMM_CODES, &nprocs);
 
-    configuration_load(argv[2], MPI_COMM_WORLD, &config);
+    configuration_load(argv[2], MPI_COMM_CODES, &config);
 
     model_net_register();
     svr_add_lp_type();
@@ -401,12 +401,12 @@ int main(
     {
         do_lp_io = 1;
         int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
-        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_WORLD);
+        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
         assert(ret == 0 || !"lp_io_prepare failure");
     }
     tw_run();
     if (do_lp_io){
-        int ret = lp_io_flush(io_handle, MPI_COMM_WORLD);
+        int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
         assert(ret == 0 || !"lp_io_flush failure");
     }
     model_net_report_stats(net_id);

src/networks/model-net/dragonfly.c

@@ -22,6 +22,11 @@
 #include "codes/quickhash.h"
 #include "codes/rc-stack.h"
 
+#ifdef ENABLE_CORTEX
+#include <cortex/cortex.h>
+#include <cortex/topology.h>
+#endif
+
 #define CREDIT_SZ 8
 #define MEAN_PROCESS 1.0
 
@@ -49,6 +54,11 @@
 #define LP_CONFIG_NM_ROUT (model_net_lp_config_names[DRAGONFLY_ROUTER])
 #define LP_METHOD_NM_ROUT (model_net_method_names[DRAGONFLY_ROUTER])
 
+#ifdef ENABLE_CORTEX
+/* This structure is defined at the end of the file */
+extern cortex_topology dragonfly_cortex_topology;
+#endif
+
 int debug_slot_count = 0;
 long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount;
 long packet_gen = 0, packet_fin = 0;
@@ -573,7 +583,7 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
     p->total_routers = p->num_groups * p->num_routers;
     p->total_terminals = p->total_routers * p->num_cn;
     int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_rank(MPI_COMM_CODES, &rank);
     if(!rank) {
         printf("\n Total nodes %d routers %d groups %d radix %d \n",
                 p->num_cn * p->total_routers, p->total_routers, p->num_groups,
@@ -599,6 +609,9 @@ static void dragonfly_configure(){
     if (anno_map->has_unanno_lp > 0){
         dragonfly_read_config(NULL, &all_params[anno_map->num_annos]);
     }
+#ifdef ENABLE_CORTEX
+	model_net_topology = dragonfly_cortex_topology;
+#endif
 }
 
 /* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */
@@ -610,20 +623,20 @@ static void dragonfly_report_stats()
    int total_minimal_packets, total_nonmin_packets;
    long total_gen, total_fin;
 
-   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
    
-   MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
+   MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
     {
-	MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
- 	MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+	MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
+ 	MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
     }
 
    /* print statistics */
@@ -3355,3 +3368,75 @@ struct model_net_method dragonfly_router_method =
     .mn_trace_register = router_register_trace,
     .mn_get_trace_type = dragonfly_get_trace_types,
 };
+
+#ifdef ENABLE_CORTEX
+
+static int dragonfly_get_number_of_compute_nodes(void* topo) {
+		// TODO
+		return -1;
+}
+
+static int dragonfly_get_number_of_routers(void* topo) {
+		// TODO
+		return -1;
+}
+
+static double dragonfly_get_router_link_bandwidth(void* topo, router_id_t r1, router_id_t r2) {
+        // TODO
+        return -1.0;
+}
+
+static double dragonfly_get_compute_node_bandwidth(void* topo, cn_id_t node) {
+        // TODO
+        return -1.0;
+}
+
+static int dragonfly_get_router_neighbor_count(void* topo, router_id_t r) {
+        // TODO
+        return 0;
+}
+
+static void dragonfly_get_router_neighbor_list(void* topo, router_id_t r, router_id_t* neighbors) {
+        // TODO
+}
+
+static int dragonfly_get_router_location(void* topo, router_id_t r, int32_t* location, int size) {
+        // TODO
+        return 0;
+}
+
+static int dragonfly_get_compute_node_location(void* topo, cn_id_t node, int32_t* location, int size) {
+        // TODO
+        return 0;
+}
+
+static router_id_t dragonfly_get_router_from_compute_node(void* topo, cn_id_t node) {
+        // TODO
+        return -1;
+}
+
+static int dragonfly_get_router_compute_node_count(void* topo, router_id_t r) {
+        // TODO
+        return 0;
+}
+
+static void dragonfly_get_router_compute_node_list(void* topo, router_id_t r, cn_id_t* nodes) {
+        // TODO
+}
+
+cortex_topology dragonfly_cortex_topology = {
+        .internal = NULL,
+		.get_number_of_routers			= dragonfly_get_number_of_routers,
+		.get_number_of_compute_nodes	= dragonfly_get_number_of_compute_nodes,
+        .get_router_link_bandwidth      = dragonfly_get_router_link_bandwidth,
+        .get_compute_node_bandwidth     = dragonfly_get_compute_node_bandwidth,
+        .get_router_neighbor_count      = dragonfly_get_router_neighbor_count,
+        .get_router_neighbor_list       = dragonfly_get_router_neighbor_list,
+        .get_router_location            = dragonfly_get_router_location,
+        .get_compute_node_location      = dragonfly_get_compute_node_location,
+        .get_router_from_compute_node   = dragonfly_get_router_from_compute_node,
+        .get_router_compute_node_count  = dragonfly_get_router_compute_node_count,
+        .get_router_compute_node_list   = dragonfly_get_router_compute_node_list,
+};
+
+#endif

src/networks/model-net/fattree.c

@@ -13,6 +13,11 @@
 #include <ctype.h>
 #include <search.h>
 
+#ifdef ENABLE_CORTEX
+#include <cortex/cortex.h>
+#include <cortex/topology.h>
+#endif
+
 #define CREDIT_SIZE 8
 #define MEAN_PROCESS 1.0
 
@@ -34,6 +39,11 @@
 #define LP_CONFIG_NM (model_net_lp_config_names[FATTREE])
 #define LP_METHOD_NM (model_net_method_names[FATTREE])
 
+#ifdef ENABLE_CORTEX
+/* This structure is defined at the end of the file */
+extern cortex_topology fattree_cortex_topology;
+#endif
+
 #if DEBUG_RC
   //Reverse Compute Debug Variables
   long long packet_event_f = 0;
@@ -546,7 +556,7 @@ static void dot_write_open_file(FILE **fout)
 
   char *dot_file_prefix = dot_file_p;
   int rank;
-  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+  MPI_Comm_rank(MPI_COMM_CODES, &rank);
   sprintf(file_name, "%s/%s.dot.%d", dir_name, dot_file_prefix, rank);
   *fout = fopen(file_name, "w+");
   if(*fout == NULL) {
@@ -891,6 +901,9 @@ static void fattree_configure(){
   if (anno_map->has_unanno_lp > 0){
     fattree_read_config(NULL, &all_params[anno_map->num_annos]);
   }
+#ifdef ENABLE_CORTEX
+  model_net_topology = fattree_cortex_topology;
+#endif
 }
 
 /* initialize a fattree compute node terminal */
@@ -1348,22 +1361,22 @@ static void fattree_report_stats()
 	long long ts_buffer_f = 0;
 	long long ts_buffer_r = 0;
 
-	MPI_Reduce( &packet_event_f, &t_packet_event_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &packet_event_r, &t_packet_event_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_generate_f, &tt_generate_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_generate_r, &tt_generate_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_send_f, &tt_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_send_r, &tt_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_arrive_f, &tt_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_arrive_r, &tt_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_buffer_f, &tt_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &t_buffer_r, &tt_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &s_send_f, &ts_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &s_send_r, &ts_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &s_arrive_f, &ts_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &s_arrive_r, &ts_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &s_buffer_f, &ts_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-	MPI_Reduce( &s_buffer_r, &ts_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
+	MPI_Reduce( &packet_event_f, &t_packet_event_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &packet_event_r, &t_packet_event_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_generate_f, &tt_generate_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_generate_r, &tt_generate_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_send_f, &tt_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_send_r, &tt_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_arrive_f, &tt_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_arrive_r, &tt_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_buffer_f, &tt_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &t_buffer_r, &tt_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &s_send_f, &ts_send_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &s_send_r, &ts_send_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &s_arrive_f, &ts_arrive_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &s_arrive_r, &ts_arrive_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &s_buffer_f, &ts_buffer_f, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+	MPI_Reduce( &s_buffer_r, &ts_buffer_r, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
 
    if(!g_tw_mynode)
    {
@@ -1399,16 +1412,16 @@ static void fattree_report_stats()
    tw_stime avg_time, max_time;
    long total_gen, total_fin;
 
-   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &fattree_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &fattree_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &fattree_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &fattree_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
 
-   MPI_Reduce( &fattree_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
-   MPI_Reduce( &fattree_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
+   MPI_Reduce( &fattree_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
+   MPI_Reduce( &fattree_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
 
    /* print statistics */
    if(!g_tw_mynode)
@@ -2836,7 +2849,7 @@ void fattree_switch_final(switch_state * s, tw_lp * lp)
 //    char *stats_file = getenv("TRACER_LINK_FILE");
 //  if(stats_file != NULL) {
     int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_rank(MPI_COMM_CODES, &rank);
     char file_name[512];
     sprintf(file_name, "%s.%d", "tracer_stats_file", rank);
     FILE *fout = fopen(file_name, "a");
@@ -2992,3 +3005,74 @@ struct model_net_method fattree_method =
   .mn_collective_call_rc = NULL
 };
 
+#ifdef ENABLE_CORTEX
+
+static int fattree_get_number_of_compute_nodes(void* topo) {
+        // TODO
+        return -1;
+}
+
+static int fattree_get_number_of_routers(void* topo) {
+        // TODO
+        return -1;
+}
+
+static double fattree_get_router_link_bandwidth(void* topo, router_id_t r1, router_id_t r2) {
+        // TODO
+        return -1.0;
+}
+
+static double fattree_get_compute_node_bandwidth(void* topo, cn_id_t node) {
+        // TODO
+        return -1.0;
+}
+
+static int fattree_get_router_neighbor_count(void* topo, router_id_t r) {
+        // TODO
+        return 0;
+}
+
+static void fattree_get_router_neighbor_list(void* topo, router_id_t r, router_id_t* neighbors) {
+        // TODO
+}
+
+static int fattree_get_router_location(void* topo, router_id_t r, int32_t* location, int size) {
+        // TODO
+        return 0;
+}
+
+static int fattree_get_compute_node_location(void* topo, cn_id_t node, int32_t* location, int size) {
+        // TODO
+        return 0;
+}
+
+static router_id_t fattree_get_router_from_compute_node(void* topo, cn_id_t node) {
+        // TODO
+        return -1;
+}
+
+static int fattree_get_router_compute_node_count(void* topo, router_id_t r) {
+        // TODO
+        return 0;
+}
+
+static void fattree_get_router_compute_node_list(void* topo, router_id_t r, cn_id_t* nodes) {
+        // TODO
+}
+
+cortex_topology fattree_cortex_topology = {
+        .internal = NULL,
+		.get_number_of_compute_nodes	= fattree_get_number_of_compute_nodes,
+		.get_number_of_routers			= fattree_get_number_of_routers,
+        .get_router_link_bandwidth      = fattree_get_router_link_bandwidth,
+        .get_compute_node_bandwidth     = fattree_get_compute_node_bandwidth,
+        .get_router_neighbor_count      = fattree_get_router_neighbor_count,
+        .get_router_neighbor_list       = fattree_get_router_neighbor_list,
+        .get_router_location            = fattree_get_router_location,
+        .get_compute_node_location      = fattree_get_compute_node_location,
+        .get_router_from_compute_node   = fattree_get_router_from_compute_node,
+        .get_router_compute_node_count  = fattree_get_router_compute_node_count,
+        .get_router_compute_node_list   = fattree_get_router_compute_node_list,
+};
+
+#endif

src/networks/model-net/model-net.c

@@ -14,6 +14,7 @@
 #include "codes/codes.h"
 #include <codes/codes_mapping.h>
 
+
 #define STR_SIZE 16
 #define PROC_TIME 10.0
 
@@ -637,6 +638,11 @@ tw_lpid model_net_find_local_device_mctx(
             sender_gid);
 }
 
+#ifdef ENABLE_CORTEX
+/* structure that gives access to the topology functions */
+cortex_topology model_net_topology;
+#endif
+
 /*
  * Local variables:
  *  c-indent-level: 4

src/networks/model-net/slimfly.c

@@ -554,7 +554,7 @@ static void slimfly_read_config(const char * anno, slimfly_param *params){
     slim_total_routers_noah = p->num_groups * p->num_routers;
     slim_total_terminals_noah = p->slim_total_routers * p->num_cn;
     int rank;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_rank(MPI_COMM_CODES, &rank);
     if(!rank) {
         printf("\n Total nodes %d total routers %d total groups %d num_terminals %d num_routers %d radix %d local_channels %d global_channels %d \n",
                 p->num_cn * p->slim_total_routers, p->slim_total_routers, p->num_groups, p->num_cn, p->num_routers,

src/networks/model-net/topology-test.c

+/* turning on track lp will generate a lot of output messages */
+#include <ross.h>
+#include <inttypes.h>
+
+#include "codes/codes-workload.h"
+#include "codes/codes.h"
+
+#ifdef ENABLE_CORTEX
+#include <cortex/topology.h>
+#endif