Merge branch 'cortex-links' into 'cortex'

Cortex links

See merge request !25

src/Makefile.subdir

@@ -179,6 +179,7 @@ src_util_templates_lp_template_dummy_main_SOURCES = \
 # get rid of annoying unused function in template
 
 bin_PROGRAMS += src/workload/codes-workload-dump
+bin_PROGRAMS += src/networks/topology-test
 bin_PROGRAMS += src/network-workloads/model-net-mpi-replay
 bin_PROGRAMS += src/network-workloads/model-net-dumpi-traces-dump
 bin_PROGRAMS += src/network-workloads/model-net-synthetic
@@ -197,7 +198,7 @@ src_network_workloads_model_net_mpi_replay_SOURCES = \
 src_network_workloads_model_net_mpi_replay_CFLAGS = $(AM_CFLAGS)
 src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c 
 src_network_workloads_model_net_synthetic_custom_dfly_SOURCES = src/network-workloads/model-net-synthetic-custom-dfly.c 
-
+src_networks_topology_test_SOURCES = src/networks/topology-test.c
 
 #bin_PROGRAMS += src/network-workload/codes-nw-test
 

src/networks/model-net/dragonfly-custom.C

@@ -26,7 +26,7 @@
 #include <cortex/topology.h>
 #endif
 
-#define DUMP_CONNECTIONS 0
+#define DUMP_CONNECTIONS 1
 #define CREDIT_SIZE 8
 #define DFLY_HASH_TABLE_SIZE 4999
 
@@ -3330,24 +3330,34 @@ static void dragonfly_custom_get_router_neighbor_list(void* topo, router_id_t r,
     int src_col = r % params->num_router_cols;
 
     /* First the routers in the same row */
-    for(int i = 0; i < params->num_router_cols; i++)
-    {
+     int i = 0;
+     int offset = 0;
+     while(i < params->num_router_cols)
+     {
        int neighbor = gid * params->num_routers + (src_row * params->num_router_cols) + i;
        if(neighbor != r)
-           neighbors[i] = neighbor;
-    }
+       {
+           neighbors[offset] = neighbor;
+           offset++;
+       }
+       i++;
+     }
 
     /* Now the routers in the same column. */
-    for(int i = 0; i <  params->num_router_rows; i++)
+    offset = 0;
+    i = 0;
+    while(i <  params->num_router_rows)
     {
         int neighbor = gid * params->num_routers + src_col + (i * params->num_router_cols);
 
-        int offset = i + params->num_router_cols - 1;
-
         if(neighbor != r)
-            neighbors[offset] = neighbor;
+        {
+            neighbors[offset+params->num_router_cols-1] = neighbor;
+            offset++;
+        }
+        i++;
     }
-    int g_offset = params->num_router_cols + params->num_router_cols - 2;
+    int g_offset = params->num_router_cols + params->num_router_rows - 2;
     
     /* Now fill up global channels */
     set<router_id_t> g_neighbors;
@@ -3412,8 +3422,9 @@ static int dragonfly_custom_get_compute_node_location(void* topo, cn_id_t node,
     if(size < 3)
         return -1;
 
-    int rid = node / params->num_cn;
-    int gid = rid / params->num_routers;
+    int rid = (node / params->num_cn) % params->num_routers;
+    int rid_global = node / params->num_cn;
+    int gid = rid_global / params->num_routers;
     int lid = node % params->num_cn;
    
     location[0] = gid;

src/networks/topology-test.c

+/* turning on track lp will generate a lot of output messages */
+#include <ross.h>
+#include <inttypes.h>
+
+#include "codes/codes-workload.h"
+#include "codes/codes.h"
+#include <cortex/topology.h>
+#include "codes/model-net.h"
+  
+/* TODO: Replace NUM_NODES and num_routers once we have functions to fetch
+ * topology node and router counts. Right now it is hard coded for Theta. */
+#define NUM_NODES 3456
+extern struct cortex_topology dragonfly_custom_cortex_topology;
+
+int main(int argc, char** argv) {
+  int* net_ids;
+  int num_nets;
+  
+  MPI_Init(&argc,&argv);
+
+  printf("\n %s ", argv[1]);
+  configuration_load(argv[1], MPI_COMM_WORLD, &config);
+   
+  model_net_register();
+  net_ids = model_net_configure(&num_nets);
+
+  /* The topo argument is NULL since we are using global variables */
+
+  /* TODO: Replace NUM_NODES and num_routers once we have functions to fetch
+   * topology node and router counts. Right now it is hard coded for Theta. */
+  int num_routers = NUM_NODES/4;
+
+  void * topo_arg = NULL;
+  double local_bandwidth, global_bandwidth, cn_bandwidth;
+  int num_router_cols, num_router_rows, num_cns;
+
+  configuration_get_value_int(&config, "PARAMS", "num_router_rows", NULL, &num_router_rows);
+  configuration_get_value_int(&config, "PARAMS", "num_cns_per_router", NULL, &num_cns);
+  configuration_get_value_int(&config, "PARAMS", "num_router_cols", NULL, &num_router_cols);
+
+  configuration_get_value_double(&config, "PARAMS", "local_bandwidth", NULL, &local_bandwidth);
+  configuration_get_value_double(&config, "PARAMS", "global_bandwidth", NULL, &global_bandwidth);
+  configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", NULL, &cn_bandwidth);
+
+  /* First check for the same rows */
+  for(int i = 0; i < num_router_cols - 1; i++)
+  {
+    double router_bw = dragonfly_custom_cortex_topology.get_router_link_bandwidth(topo_arg, i, i+1);
+    assert(router_bw == local_bandwidth); 
+  }
+
+  /* Now check for black links, connections in the same column. */
+  for(int i = 0; i < num_router_rows - 1; i++)
+  {
+    int src = i * num_router_cols;
+    int dest = (i + 1) * num_router_cols;  
+    double router_bw = dragonfly_custom_cortex_topology.get_router_link_bandwidth(topo_arg, src, dest);
+    assert(router_bw == local_bandwidth);
+  }
+
+  /* Now check for a router in a different row and column, should not have a
+   * connection in between them */
+  for(int i = 0; i < num_router_rows - 1; i++)
+  {
+    int src = i * num_router_cols;
+    int dest = (i + 1) * num_router_cols + 2;  
+    double router_bw = dragonfly_custom_cortex_topology.get_router_link_bandwidth(topo_arg, src, dest);
+    assert(router_bw == 0.0);
+  }
+
+  /* For global connections, check routers 0-3 connections to groups 1 and 5 */
+  for(int i = 0; i < 4; i++)
+  {
+    int dest = i + (num_router_rows * num_router_cols); 
+    double router_bw = dragonfly_custom_cortex_topology.get_router_link_bandwidth(topo_arg, i, dest);
+    assert(router_bw == global_bandwidth);
+
+    dest = i + (5 * num_router_rows * num_router_cols);
+    router_bw = dragonfly_custom_cortex_topology.get_router_link_bandwidth(topo_arg, i, dest);
+    assert(router_bw == global_bandwidth);
+  }
+
+  /* For the first four compute nodes , get their compute node bandwidth now */
+  for(int i = 0; i < 4; i++)
+  {
+    double cn_bw = dragonfly_custom_cortex_topology.get_compute_node_bandwidth(topo_arg, i);
+    assert(cn_bw == cn_bandwidth);
+
+    cn_bw = dragonfly_custom_cortex_topology.get_compute_node_bandwidth(topo_arg, -1);
+    assert(cn_bw == -1.0);
+
+    cn_bw = dragonfly_custom_cortex_topology.get_compute_node_bandwidth(topo_arg, 3800);
+    assert(cn_bw == -1.0);
+
+  }
+
+  int grp1_offset = num_router_cols * num_router_rows;
+  /* Check connections for the first router row */
+  for(int i = grp1_offset; i < grp1_offset + num_router_cols; i++)
+  {
+      int num_neighbors = dragonfly_custom_cortex_topology.get_router_neighbor_count(topo_arg, i);
+      //printf("\n Router %d Number of neighbors %d ", i, num_neighbors);
+      assert(num_neighbors == 22);
+
+      router_id_t * routers = malloc(num_neighbors * sizeof(router_id_t));
+      dragonfly_custom_cortex_topology.get_router_neighbor_list(topo_arg, i, routers);
+
+      if(i == 0)
+      {
+        for(int j = 0; j < num_neighbors; j++)
+          printf("\n Router id %d ", routers[j]);
+
+      }
+        int32_t * location = malloc(2 * sizeof(int32_t));
+        dragonfly_custom_cortex_topology.get_router_location(topo_arg, i, location, sizeof(location));
+
+        assert(location[0] == 1 && location[1] == (i - grp1_offset));
+
+        free(location);
+
+        for(int k = 0; k < num_cns; k++)
+        {
+            cn_id_t cn_id = i * num_cns + k;
+            int32_t * loc_cn = malloc(3 * sizeof(int32_t));
+            dragonfly_custom_cortex_topology.get_compute_node_location(topo_arg, cn_id, loc_cn, sizeof(loc_cn));
+            //printf("\n location[0] %d location[1] %d location[2] %d ", location[0], location[1], location[2]);
+            assert(location[0] == 1 && location[1] == (i - grp1_offset) && location[2] == k);
+            free(loc_cn);
+
+            router_id_t rid = dragonfly_custom_cortex_topology.get_router_from_compute_node(topo_arg, cn_id);
+            assert(rid == i);
+        }
+
+        int cn_count = dragonfly_custom_cortex_topology.get_router_compute_node_count(topo_arg, i);
+        cn_id_t * cn_ids = malloc(sizeof(cn_id_t) * cn_count);
+        dragonfly_custom_cortex_topology.get_router_compute_node_list(topo_arg, i, cn_ids);
+
+        for(int k = 0; k < cn_count; k++)
+        {
+           cn_id_t cn_id = i * num_cns + k;
+           assert(cn_ids[k] == cn_id);
+        }
+  }
+  MPI_Finalize();
+}