updates to dragonfly custom: selecting one path among multiple available to...

updates to dragonfly custom: selecting one path among multiple available to avoid intra-group congestion, enabling multiple connections within a router row, fixing lp-io radix bug

scripts/gen-cray-topo/connections_general.c

@@ -74,7 +74,7 @@ int main(int argc, char **argv) {
         } else {
           ndstg--;
         }
-        int gsize = 2, gs = 16;
+        int gsize = 2, gs = c;
         for(int row = 0; row < r; row++) {
           int srcrB = srcg * r * c + row * c, srcr;
           int dstrB = dstg * r * c + row * c, dstr;
@@ -90,11 +90,11 @@ int main(int argc, char **argv) {
                 for(int block = 0; block < gsize; block++) {
                 fwrite(&srcr, sizeof(int), 1, inter);
                 fwrite(&dstr, sizeof(int), 1, inter);
-                //printf("INTER %d %d srcg %d destg %d srcrb %d dstrB %d \n", srcr, dstr, srcg, dstg, srcrB, dstrB);
+                printf("INTER %d %d srcg %d destg %d srcrb %d dstrB %d \n", srcr, dstr, srcg, dstg, srcrB, dstrB);
             }
-           }
             srcr++;
             dstr++;
+            }
         }
       }
     }

scripts/gen-cray-topo/connections_general_patched.c

+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
+// Produced at the Lawrence Livermore National Laboratory.
+//
+// Written by:
+//     Nikhil Jain <nikhil.jain@acm.org>
+//     Abhinav Bhatele <bhatele@llnl.gov>
+//     Peer-Timo Bremer <ptbremer@llnl.gov>
+//
+// LLNL-CODE-678961. All rights reserved.
+//
+// This file is part of Damselfly. For details, see:
+// https://github.com/scalability-llnl/damselfly
+// Please also read the LICENSE file for our notice and the LGPL.
+//////////////////////////////////////////////////////////////////////////////
+
+#include "stdio.h"
+#include "stdlib.h"
+
+//Usage ./binary num_groups num_rows num_columns intra_file inter_file
+
+int main(int argc, char **argv) {
+  if(argc < 3) {
+    printf("Correct usage: %s <num_group> <num_rows> <num_cols> <cons_across_groups> <cons_in_row> <cons_in_col> <intra_file> <inter_file>", argv[0]);
+    exit(0);
+  }
+
+  int g = atoi(argv[1]);
+  int r = atoi(argv[2]);
+  int c = atoi(argv[3]);
+  int g_p = atoi(argv[4]);
+  int r_p = atoi(argv[5]);
+  int c_p = atoi(argv[6]);
+
+  int total_routers = g * r * c; 
+  int routers_per_g = r * c; 
+
+  FILE *intra = fopen(argv[7], "wb");
+  FILE *inter = fopen(argv[8], "wb");
+
+  int router = 0;
+  int green = 0, black = 1;
+  int groups = 0;
+  for(int rows = 0; rows < r; rows++) {
+    for(int cols = 0; cols < c; cols++) {
+      for(int cols1 = 0; cols1 < c; cols1++) {
+        if(cols1 != cols) {
+          int dest = (rows * c) + cols1;
+          for(int link = 0; link < c_p; link++) {
+            fwrite(&router, sizeof(int), 1, intra);
+            fwrite(&dest, sizeof(int), 1, intra);
+            fwrite(&green, sizeof(int), 1, intra);
+            printf("INTRA %d %d %d\n", router, dest, green);
+          }
+        }
+      }
+      for(int rows1 = 0; rows1 < r; rows1++) {
+        if(rows1 != rows) {
+          int dest = (rows1 * c) + cols;
+          for(int link = 0; link < r_p; link++) {
+            fwrite(&router, sizeof(int), 1, intra);
+            fwrite(&dest, sizeof(int), 1, intra);
+            fwrite(&black, sizeof(int), 1, intra);
+            printf("INTRA %d %d %d\n", router, dest, black);
+          }
+        }
+      }
+      router++;
+    }
+  }
+
+  for(int srcg = 0; srcg < g; srcg++) {
+    for(int dstg = 0; dstg < g; dstg++) {
+      if(srcg != dstg) {
+        int nsrcg = srcg;
+        int ndstg = dstg;
+        if(srcg > dstg) {
+          nsrcg--;
+        } else {
+          ndstg--;
+        }
+        int startSrc = ndstg * g_p;
+        int startDst = nsrcg * g_p;
+        for(int link = 0; link < g_p; link++) {
+          int srcrB = srcg * routers_per_g, srcr;
+          int dstrB = dstg * routers_per_g, dstr;
+          srcr = srcrB + (startSrc + link) % routers_per_g;
+          dstr = dstrB + (startDst + link) % routers_per_g;
+
+          if(srcr >= total_routers || dstr >= total_routers)
+            printf("\n connection between invalid routers src %d and dest %d ", srcr, dstr);
+
+          fwrite(&srcr, sizeof(int), 1, inter);
+          fwrite(&dstr, sizeof(int), 1, inter);
+          printf("INTER %d %d srcg %d destg %d\n", srcr, dstr, srcg, dstg);
+        }
+      }
+    }
+  }
+
+  fclose(intra);
+  fclose(inter);
+}

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf

@@ -2,11 +2,11 @@ LPGROUPS
 {
    MODELNET_GRP
    {
-      repetitions="1600";
+      repetitions="1520";
 # name of this lp changes according to the model
-      nw-lp="4";
+      nw-lp="8";
 # these lp names will be the same for dragonfly-custom model
-      modelnet_dragonfly_custom="4";
+      modelnet_dragonfly_custom="8";
       modelnet_dragonfly_custom_router="1";
    }
 }
@@ -24,11 +24,11 @@ PARAMS
    # number of routers within each group
    # this is dictated by the dragonfly configuration files
    # intra-group rows for routers
-   num_router_rows="4";
+   num_router_rows="1";
    # intra-group columns for routers
-   num_router_cols="20";
+   num_router_cols="40";
    # number of groups in the network
-   num_groups="20";
+   num_groups="38";
 # buffer size in bytes for local virtual channels 
    local_vc_size="8192";
 #buffer size in bytes for global virtual channels 
@@ -38,20 +38,24 @@ PARAMS
 #bandwidth in GiB/s for local channels 
    local_bandwidth="5.25";
 # bandwidth in GiB/s for global channels 
-   global_bandwidth="18.75";
+   global_bandwidth="4.69";
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="8.0";
+# Number of row channels
+   num_row_chans="2";
+# Number of column channels
+   num_col_chans="1";
 # ROSS message size 
    message_size="592";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
-   num_cns_per_router="4";
+   num_cns_per_router="8";
 # number of global channels per router 
-   num_global_channels="10";
+   num_global_channels="4";
 # network config file for intra-group connections 
-   intra-group-connections="../src/network-workloads/conf/dragonfly-custom/intra-custom-upd";
+   intra-group-connections="/Users/mmubarak/Documents/software_development/codes/scripts/gen-cray-topo/intratest";
 # network config file for inter-group connections
-   inter-group-connections="../src/network-workloads/conf/dragonfly-custom/inter-custom-upd";
+   inter-group-connections="/Users/mmubarak/Documents/software_development/codes/scripts/gen-cray-topo/intertest";
 # routing protocol to be used 
    routing="prog-adaptive";
 }