Merge branch 'dragonfly-validation' into 'master'

Dragonfly validation

See merge request !26

CONTRIBUTORS.md

@@ -3,8 +3,10 @@ Nikhil Jain, Abhinav Bhatele (LLNL)
     - Addition of direct scheme for setting up dragonfly network topology.
     - Network configuration setup for custom dragonfly model.
     - Topology generations scripts for custom dragonfly model.
+    - Bug fix for virtual channel deadlocks in custom dragonfly model.
     - Bug reporter for CODES network models.
     - Fat tree network setup and adaptive routing.
+    - Pending: Merging Express mesh model to master.
 
 Jens Domke (U. of Dresden)
     - Static routing in fat tree network model.

codes/model-net-lp.h

@@ -22,6 +22,7 @@ extern "C" {
 #include "model-net.h"
 #include "model-net-sched.h"
 #include "net/dragonfly.h"
+#include "net/dragonfly-custom.h"
 #include "net/slimfly.h"
 #include "net/fattree.h"
 #include "net/loggp.h"
@@ -124,6 +125,7 @@ typedef struct model_net_wrap_msg {
     union {
         model_net_base_msg      m_base;  // base lp
         terminal_message        m_dfly;  // dragonfly
+        terminal_custom_message        m_custom_dfly;  // dragonfly-custom
         slim_terminal_message   m_slim;  // slimfly
 	fattree_message		m_fat;   // fattree
         loggp_message           m_loggp; // loggp

scripts/allocation_gen/config_alloc.conf

 rand
-792
-200
+8316
+2048 2048
 

scripts/gen-cray-topo/connections_general.c

@@ -29,12 +29,15 @@ int main(int argc, char **argv) {
   int r = atoi(argv[2]);
   int c = atoi(argv[3]);
 
+  int total_routers = g * r * c; 
+
   FILE *intra = fopen(argv[4], "wb");
   FILE *inter = fopen(argv[5], "wb");
 
   int router = 0;
   int green = 0, black = 1;
   int groups = 0;
+  printf("\n Rows %d Cols %d Groups %d ", r, c, g);
   for(int rows = 0; rows < r; rows++) {
     for(int cols = 0; cols < c; cols++) {
       for(int cols1 = 0; cols1 < c; cols1++) {
@@ -71,7 +74,7 @@ int main(int argc, char **argv) {
         } else {
           ndstg--;
         }
-        int gsize = 2, gs = 16;
+        int gsize = 2, gs = c;
         for(int row = 0; row < r; row++) {
           int srcrB = srcg * r * c + row * c, srcr;
           int dstrB = dstg * r * c + row * c, dstr;
@@ -79,15 +82,19 @@ int main(int argc, char **argv) {
           int dstB = (nsrcg % (gs/2)) * 2;
           srcr = srcrB + srcB;
           dstr = dstrB + dstB;
-          for(int r = 0; r < 2; r++) {
-            for(int block = 0; block < gsize; block++) {
-              fwrite(&srcr, sizeof(int), 1, inter);
-              fwrite(&dstr, sizeof(int), 1, inter);
-              printf("INTER %d %d\n", srcr, dstr);
+
+          if(srcr >= total_routers || dstr >= total_routers)
+              printf("\n connection between invalid routers src %d and dest %d ", srcr, dstr);
+
+            for(int r = 0; r < 2; r++) {
+                for(int block = 0; block < gsize; block++) {
+                fwrite(&srcr, sizeof(int), 1, inter);
+                fwrite(&dstr, sizeof(int), 1, inter);
+                printf("INTER %d %d srcg %d destg %d srcrb %d dstrB %d \n", srcr, dstr, srcg, dstg, srcrB, dstrB);
             }
             srcr++;
             dstr++;
-          }
+            }
         }
       }
     }

scripts/gen-cray-topo/connections_general_patched.c

+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
+// Produced at the Lawrence Livermore National Laboratory.
+//
+// Written by:
+//     Nikhil Jain <nikhil.jain@acm.org>
+//     Abhinav Bhatele <bhatele@llnl.gov>
+//     Peer-Timo Bremer <ptbremer@llnl.gov>
+//
+// LLNL-CODE-678961. All rights reserved.
+//
+// This file is part of Damselfly. For details, see:
+// https://github.com/scalability-llnl/damselfly
+// Please also read the LICENSE file for our notice and the LGPL.
+//////////////////////////////////////////////////////////////////////////////
+
+#include "stdio.h"
+#include "stdlib.h"
+
+//Usage ./binary num_groups num_rows num_columns intra_file inter_file
+
+int main(int argc, char **argv) {
+  if(argc < 3) {
+    printf("Correct usage: %s <num_group> <num_rows> <num_cols> <cons_across_groups> <cons_in_row> <cons_in_col> <intra_file> <inter_file>", argv[0]);
+    exit(0);
+  }
+
+  int g = atoi(argv[1]);
+  int r = atoi(argv[2]);
+  int c = atoi(argv[3]);
+  int g_p = atoi(argv[4]);
+  int r_p = atoi(argv[5]);
+  int c_p = atoi(argv[6]);
+
+  int total_routers = g * r * c; 
+  int routers_per_g = r * c; 
+
+  FILE *intra = fopen(argv[7], "wb");
+  FILE *inter = fopen(argv[8], "wb");
+
+  int router = 0;
+  int green = 0, black = 1;
+  int groups = 0;
+  for(int rows = 0; rows < r; rows++) {
+    for(int cols = 0; cols < c; cols++) {
+      for(int cols1 = 0; cols1 < c; cols1++) {
+        if(cols1 != cols) {
+          int dest = (rows * c) + cols1;
+          for(int link = 0; link < c_p; link++) {
+            fwrite(&router, sizeof(int), 1, intra);
+            fwrite(&dest, sizeof(int), 1, intra);
+            fwrite(&green, sizeof(int), 1, intra);
+            printf("INTRA %d %d %d\n", router, dest, green);
+          }
+        }
+      }
+      for(int rows1 = 0; rows1 < r; rows1++) {
+        if(rows1 != rows) {
+          int dest = (rows1 * c) + cols;
+          for(int link = 0; link < r_p; link++) {
+            fwrite(&router, sizeof(int), 1, intra);
+            fwrite(&dest, sizeof(int), 1, intra);
+            fwrite(&black, sizeof(int), 1, intra);
+            printf("INTRA %d %d %d\n", router, dest, black);
+          }
+        }
+      }
+      router++;
+    }
+  }
+
+  for(int srcg = 0; srcg < g; srcg++) {
+    for(int dstg = 0; dstg < g; dstg++) {
+      if(srcg != dstg) {
+        int nsrcg = srcg;
+        int ndstg = dstg;
+        if(srcg > dstg) {
+          nsrcg--;
+        } else {
+          ndstg--;
+        }
+        int startSrc = ndstg * g_p;
+        int startDst = nsrcg * g_p;
+        for(int link = 0; link < g_p; link++) {
+          int srcrB = srcg * routers_per_g, srcr;
+          int dstrB = dstg * routers_per_g, dstr;
+          srcr = srcrB + (startSrc + link) % routers_per_g;
+          dstr = dstrB + (startDst + link) % routers_per_g;
+
+          if(srcr >= total_routers || dstr >= total_routers)
+            printf("\n connection between invalid routers src %d and dest %d ", srcr, dstr);
+
+          fwrite(&srcr, sizeof(int), 1, inter);
+          fwrite(&dstr, sizeof(int), 1, inter);
+          printf("INTER %d %d srcg %d destg %d\n", srcr, dstr, srcg, dstg);
+        }
+      }
+    }
+  }
+
+  fclose(intra);
+  fclose(inter);
+}

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom-768-nodes.conf.in

@@ -49,9 +49,9 @@ PARAMS
 # number of global channels per router 
    num_global_channels="10";
 # network config file for intra-group connections 
-   intra-group-connections="@abs_srcdir@/intra-custom-small";
+   intra-group-connections="@abs_srcdir@/intra-theta";
 # network config file for inter-group connections
-   inter-group-connections="@abs_srcdir@/inter-custom-small";
+   inter-group-connections="@abs_srcdir@/inter-theta";
 # routing protocol to be used 
    routing="prog-adaptive";
 }

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf

@@ -2,11 +2,11 @@ LPGROUPS
 {
    MODELNET_GRP
    {
-      repetitions="1600";
+      repetitions="1520";
 # name of this lp changes according to the model
-      nw-lp="4";
+      nw-lp="8";
 # these lp names will be the same for dragonfly-custom model
-      modelnet_dragonfly_custom="4";
+      modelnet_dragonfly_custom="8";
       modelnet_dragonfly_custom_router="1";
    }
 }
@@ -24,11 +24,11 @@ PARAMS
    # number of routers within each group
    # this is dictated by the dragonfly configuration files
    # intra-group rows for routers
-   num_router_rows="4";
+   num_router_rows="1";
    # intra-group columns for routers
-   num_router_cols="20";
+   num_router_cols="40";
    # number of groups in the network
-   num_groups="20";
+   num_groups="38";
 # buffer size in bytes for local virtual channels 
    local_vc_size="8192";
 #buffer size in bytes for global virtual channels 
@@ -38,20 +38,24 @@ PARAMS
 #bandwidth in GiB/s for local channels 
    local_bandwidth="5.25";
 # bandwidth in GiB/s for global channels 
-   global_bandwidth="18.75";
+   global_bandwidth="4.69";
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="8.0";
+# Number of row channels
+   num_row_chans="2";
+# Number of column channels
+   num_col_chans="1";
 # ROSS message size 
    message_size="592";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
-   num_cns_per_router="4";
+   num_cns_per_router="8";
 # number of global channels per router 
-   num_global_channels="10";
+   num_global_channels="4";
 # network config file for intra-group connections 
-   intra-group-connections="../src/network-workloads/conf/dragonfly-custom/intra-custom";
+   intra-group-connections="/Users/mmubarak/Documents/software_development/codes/scripts/gen-cray-topo/intratest";
 # network config file for inter-group connections
-   inter-group-connections="../src/network-workloads/conf/dragonfly-custom/inter-custom";
+   inter-group-connections="/Users/mmubarak/Documents/software_development/codes/scripts/gen-cray-topo/intertest";
 # routing protocol to be used 
    routing="prog-adaptive";
 }

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf

@@ -37,7 +37,7 @@ PARAMS
 #bandwidth in GiB/s for local channels 
    local_bandwidth="5.25";
 # bandwidth in GiB/s for global channels 
-   global_bandwidth="18.75";
+   global_bandwidth="4.69";
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="16.0";
 # ROSS message size 
@@ -46,7 +46,7 @@ PARAMS
 # file
    num_cns_per_router="4";
 # number of global channels per router 
-   num_global_channels="10";
+   num_global_channels="4";
 # network config file for intra-group connections 
    intra-group-connections="../src/network-workloads/conf/dragonfly-custom/intra-theta";
 # network config file for inter-group connections

src/network-workloads/model-net-synthetic-custom-dfly.c

@@ -35,8 +35,8 @@ static lp_io_handle io_handle;
 static unsigned int lp_io_use_suffix = 0;
 static int do_lp_io = 0;
 static int num_msgs = 20;
-static unsigned int sampling_interval = 800000;
-static unsigned int sampling_end_time = 1600000;
+static tw_stime sampling_interval = 800000;
+static tw_stime sampling_end_time = 1600000;
 
 typedef struct svr_msg svr_msg;
 typedef struct svr_state svr_state;
@@ -111,8 +111,8 @@ const tw_optdef app_opt [] =
         TWOPT_GROUP("Model net synthetic traffic " ),
     	TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "),
     	TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "),
-    	TWOPT_UINT("sampling-interval", sampling_interval, "the sampling interval "),
-    	TWOPT_UINT("sampling-end-time", sampling_end_time, "sampling end time "),
+    	TWOPT_STIME("sampling-interval", sampling_interval, "the sampling interval "),
+    	TWOPT_STIME("sampling-end-time", sampling_end_time, "sampling end time "),
 	    TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"),
         TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"),
         TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"),

src/network-workloads/model-net-synthetic.c

@@ -35,8 +35,8 @@ static lp_io_handle io_handle;
 static unsigned int lp_io_use_suffix = 0;
 static int do_lp_io = 0;
 static int num_msgs = 20;
-static unsigned int sampling_interval = 800000;
-static unsigned int sampling_end_time = 1600000;
+static tw_stime sampling_interval = 800000;
+static tw_stime sampling_end_time = 1600000;
 
 typedef struct svr_msg svr_msg;
 typedef struct svr_state svr_state;
@@ -139,8 +139,8 @@ const tw_optdef app_opt [] =
         TWOPT_GROUP("Model net synthetic traffic " ),
     	TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "),
     	TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "),
-    	TWOPT_UINT("sampling-interval", sampling_interval, "the sampling interval "),
-    	TWOPT_UINT("sampling-end-time", sampling_end_time, "sampling end time "),
+    	TWOPT_STIME("sampling-interval", sampling_interval, "the sampling interval "),
+    	TWOPT_STIME("sampling-end-time", sampling_end_time, "sampling end time "),
 	    TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"),
         TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"),
         TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"),

src/networks/model-net/model-net-lp.c

@@ -18,6 +18,7 @@
 /**** BEGIN SIMULATION DATA STRUCTURES ****/
 
 int model_net_base_magic;
+int mn_sample_enabled = 0;
 
 // message-type specific offsets - don't want to get bitten later by alignment
 // issues...
@@ -36,7 +37,6 @@ static const char              * annos[CONFIGURATION_MAX_ANNOS];
 static model_net_base_params     all_params[CONFIGURATION_MAX_ANNOS];
 
 static tw_stime mn_sample_interval = 0.0;
-static int mn_sample_enabled = 0;
 static tw_stime mn_sample_end = 0.0;
 
 typedef struct model_net_base_state {
@@ -283,9 +283,9 @@ void model_net_base_configure(){
     msg_offsets[DRAGONFLY_ROUTER] =
         offsetof(model_net_wrap_msg, msg.m_dfly);
     msg_offsets[DRAGONFLY_CUSTOM] =
-        offsetof(model_net_wrap_msg, msg.m_dfly);
+        offsetof(model_net_wrap_msg, msg.m_custom_dfly);
     msg_offsets[DRAGONFLY_CUSTOM_ROUTER] =
-        offsetof(model_net_wrap_msg, msg.m_dfly);
+        offsetof(model_net_wrap_msg, msg.m_custom_dfly);
     msg_offsets[SLIMFLY] =
         offsetof(model_net_wrap_msg, msg.m_slim);
     msg_offsets[FATTREE] =

src/networks/model-net/torus.c

@@ -1694,7 +1694,7 @@ static void torus_report_stats()
 
     if(!g_tw_mynode)
      {
-       printf(" Average number of hops traversed %f average message latency %lf us maximum message latency %lf us finished packets %lld finished hops %lld \n",
+       printf(" Average number of hops traversed %f average packet latency %lf us maximum packet latency %lf us finished packets %lld finished hops %lld \n",
                (float)avg_hops/total_finished_packets, avg_time/(total_finished_packets*1000), max_time/1000, total_finished_packets, avg_hops);
      }
 }

tests/Makefile.subdir

@@ -34,6 +34,7 @@ TESTS += tests/lp-io-test.sh \
  tests/modelnet-test-slimfly.sh \
  tests/modelnet-test-dragonfly-synthetic.sh \
  tests/modelnet-test-dragonfly-traces.sh \
+ tests/modelnet-test-dragonfly-custom-synthetic.sh \
  tests/modelnet-test-dragonfly-custom-traces.sh \
  tests/modelnet-test-fattree-synthetic.sh \
  tests/modelnet-test-slimfly-synthetic.sh \
@@ -61,10 +62,12 @@ EXTRA_DIST += tests/lp-io-test.sh \
  tests/modelnet-test.sh \
  tests/modelnet-test-torus.sh \
  tests/modelnet-test-torus-traces.sh \
- tests/modelnet-test-dragonfly-custom-traces.sh \
  tests/modelnet-test-loggp.sh \
  tests/modelnet-test-dragonfly.sh \
+ tests/modelnet-test-dragonfly-synthetic.sh \
  tests/modelnet-test-dragonfly-traces.sh \
+ tests/modelnet-test-dragonfly-custom-synthetic.sh \
+ tests/modelnet-test-dragonfly-custom-traces.sh \
  tests/modelnet-test-slimfly.sh \
  tests/modelnet-test-slimfly-synthetic.sh \
  tests/modelnet-test-slimfly-traces.sh \
@@ -115,4 +118,4 @@ tests_modelnet_simplep2p_test_SOURCES = tests/modelnet-simplep2p-test.c
 tests_modelnet_p2p_bw_SOURCES = tests/modelnet-p2p-bw.c
 tests_concurrent_msg_recv_SOURCES = tests/concurrent-msg-recv.c
 tests_modelnet_test_collective_SOURCES = tests/modelnet-test-collective.c
-tests_modelnet_prio_sched_test_SOURCES = tests/modelnet-prio-sched-test.c
+tests_modelnet_prio_sched_test_SOURCES = tests/modelnet-prio-sched-test.c
\ No newline at end of file

tests/modelnet-test-dragonfly-synthetic.sh

 #!/bin/bash
-src/network-workloads/model-net-synthetic --sync=1 --num_messages=1 -- $srcdir/src/network-workloads/conf/modelnet-synthetic-dragonfly.conf 
 
+src/network-workloads/model-net-synthetic --sync=1 --num_messages=1 -- $srcdir/src/network-workloads/conf/modelnet-synthetic-dragonfly.conf