Adding theta network configuration

scripts/gen-cray-topo/README.txt

@@ -8,6 +8,13 @@
     translates Edison's network configuration into a file format that can be fed
     into the simulation.]
 
+- Theta network config files:
+
+   python theta.py theta.interconnect intra-theta inter-theta
+
+   [intra-theta and inter-theta are the intra and inter-group config files for
+   dragonfly. ]
+
  ** Generating customizable dragonfly interconnects **:
 
 mpicc connections_general.c -o connections_general

scripts/gen-cray-topo/theta.py

+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2014, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# Written by:
+#     Nikhil Jain <nikhil.jain@acm.org>
+#     Abhinav Bhatele <bhatele@llnl.gov>
+#     Peer-Timo Bremer <ptbremer@llnl.gov>
+#
+# LLNL-CODE-678961. All rights reserved.
+#
+# This file is part of Damselfly. For details, see:
+# https://github.com/LLNL/damselfly
+# Please also read the LICENSE file for our notice and the LGPL.
+##############################################################################
+
+# Modified by CODES team on November 3rd to support theta configuration format -- MM
+
+import sys
+import re
+import numpy as np
+import struct
+
+filename = sys.argv[1]
+intracon = open(sys.argv[2], "wb")
+intercon = open(sys.argv[3], "wb")
+
+def router(group, row, col):
+    return group*96 + row*16 + col
+
+numblack = np.zeros((864,864), dtype=np.int)
+numblue = np.zeros((864,864), dtype=np.int)
+
+with open(filename) as ofile:
+    matches =    re.findall('c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\).(\w+).->.c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\)', ofile.read(), re.MULTILINE)
+
+print matches 
+
+for match in matches:
+    srcgrp = int(match[0])
+
+    #if(srcgrp > 12):
+	#srcgrp = srcgrp - 1
+    srcrow = int(match[1])
+    srccol = int(match[2])
+    srcrouter = router(srcgrp, srcrow, srccol)
+
+    print srcrouter
+
+    color = match[3]
+
+    dstgrp = int(match[4])
+    #if(dstgrp > 12):
+	#dstgrp = dstgrp - 1
+    dstrow = int(match[5])
+    dstcol = int(match[6])
+    dstrouter = router(dstgrp, dstrow, dstcol)
+
+    # count number of black and blue links per router pair
+    if color == 'black':
+	numblack[srcrouter][dstrouter] += 1
+    if color == 'blue':
+	numblue[srcrouter][dstrouter] += 1
+
+    if srcgrp == 0:
+	if color == 'blue':
+	    # write to inter-con file
+	    intercon.write(struct.pack('2i', srcrouter, dstrouter))
+	    # print 'BLUE', srcrouter, dstrouter
+	else:
+	    # write to intra-con file
+	    if color == 'green':
+		intracon.write(struct.pack('3i', srcrouter, dstrouter, 0))
+		# print 'GREEN', srcrouter, dstrouter, 0
+	    elif numblack[srcrouter][dstrouter] < 4:
+		intracon.write(struct.pack('3i', srcrouter, dstrouter, 1))
+		# print 'BLACK', srcrouter, dstrouter, 1
+    else:
+	if color == 'blue':
+	    # only write the inter-con file
+	    intercon.write(struct.pack('2i', srcrouter, dstrouter))
+	    # print 'BLUE', srcrouter, dstrouter
+
+#for i in range(0, 864):
+#    for j in range(0, 864):
+#	if(numblack[i][j] != 0):
+#	    print numblack[i][j],
+
+#print "\n"
+#for i in range(0, 864):
+#    for j in range(0, 864):
+#	if(numblue[i][j] != 0):
+#	    print numblue[i][j],
+
+intracon.close()
+intercon.close()
+

src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-custom.conf

-LPGROUPS
-{
-   MODELNET_GRP
-   {
-      repetitions="1600";
-# name of this lp changes according to the model
-      server="4";
-# these lp names will be the same for dragonfly-custom model
-      modelnet_dragonfly_custom="4";
-      modelnet_dragonfly_custom_router="1";
-   }
-}
-PARAMS
-{
-# packet size in the network 
-   packet_size="1024";
-   modelnet_order=( "dragonfly_custom","dragonfly_custom_router" );
-   # scheduler options
-   modelnet_scheduler="fcfs";
-# chunk size in the network (when chunk size = packet size, packets will not be
-# divided into chunks)
-   chunk_size="1024";
-   # modelnet_scheduler="round-robin";
-   # number of routers within each group
-   # this is dictated by the dragonfly configuration files
-   num_routers="80";
-   # number of groups in the network
-   num_groups="20";
-# buffer size in bytes for local virtual channels 
-   local_vc_size="8192";
-#buffer size in bytes for global virtual channels 
-   global_vc_size="16384";
-#buffer size in bytes for compute node virtual channels 
-   cn_vc_size="8192";
-#bandwidth in GiB/s for local channels 
-   local_bandwidth="5.25";
-# bandwidth in GiB/s for global channels 
-   global_bandwidth="18.75";
-# bandwidth in GiB/s for compute node-router channels 
-   cn_bandwidth="8.0";
-# ROSS message size 
-   message_size="584";
-# number of compute nodes connected to router, dictated by dragonfly config
-# file
-   num_cns_per_router="4";
-# number of global channels per router 
-   num_global_channels="8";
-# network config file for intra-group connections 
-   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom";
-# network config file for inter-group connections
-   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom";
-# routing protocol to be used 
-   routing="adaptive";
-}

src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-edison.conf

-LPGROUPS
-{
-   MODELNET_GRP
-   {
-      repetitions="1440";
-# name of this lp changes according to the model
-      server="4";
-# these lp names will be the same for dragonfly-custom model
-      modelnet_dragonfly_custom="4";
-      modelnet_dragonfly_custom_router="1";
-   }
-}
-PARAMS
-{
-# packet size in the network 
-   packet_size="1024";
-   modelnet_order=( "dragonfly_custom","dragonfly_custom_router" );
-   # scheduler options
-   modelnet_scheduler="fcfs";
-# chunk size in the network (when chunk size = packet size, packets will not be
-# divided into chunks)
-   chunk_size="1024";
-   # modelnet_scheduler="round-robin";
-   # number of routers within each group
-   # this is dictated by the dragonfly configuration files
-   num_routers="96";
-   # number of groups in the network
-   num_groups="15";
-# buffer size in bytes for local virtual channels 
-   local_vc_size="8192";
-#buffer size in bytes for global virtual channels 
-   global_vc_size="16384";
-#buffer size in bytes for compute node virtual channels 
-   cn_vc_size="8192";
-#bandwidth in GiB/s for local channels 
-   local_bandwidth="5.25";
-# bandwidth in GiB/s for global channels 
-   global_bandwidth="18.75";
-# bandwidth in GiB/s for compute node-router channels 
-   cn_bandwidth="8.0";
-# ROSS message size 
-   message_size="584";
-# number of compute nodes connected to router, dictated by dragonfly config
-# file
-   num_cns_per_router="4";
-# number of global channels per router 
-   num_global_channels="8";
-# network config file for intra-group connections 
-   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison";
-# network config file for inter-group connections
-   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison";
-# routing protocol to be used 
-   routing="adaptive";
-}

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf

@@ -2,7 +2,7 @@ LPGROUPS
 {
    MODELNET_GRP
    {
-      repetitions="3200";
+      repetitions="1600";
 # name of this lp changes according to the model
       nw-lp="4";
 # these lp names will be the same for dragonfly-custom model
@@ -23,9 +23,9 @@ PARAMS
    # modelnet_scheduler="round-robin";
    # number of routers within each group
    # this is dictated by the dragonfly configuration files
-   num_routers="128";
+   num_routers="80";
    # number of groups in the network
-   num_groups="25";
+   num_groups="20";
 # buffer size in bytes for local virtual channels 
    local_vc_size="8192";
 #buffer size in bytes for global virtual channels 
@@ -39,16 +39,16 @@ PARAMS
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="8.0";
 # ROSS message size 
-   message_size="584";
+   message_size="592";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="4";
 # number of global channels per router 
-   num_global_channels="8";
+   num_global_channels="10";
 # network config file for intra-group connections 
-   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom";
+   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/intra-custom";
 # network config file for inter-group connections
-   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom";
+   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/inter-custom";
 # routing protocol to be used 
    routing="prog-adaptive";
 }

src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-edison.conf

@@ -39,16 +39,16 @@ PARAMS
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="8.0";
 # ROSS message size 
-   message_size="584";
+   message_size="592";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="4";
 # number of global channels per router 
    num_global_channels="8";
 # network config file for intra-group connections 
-   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison";
+   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/intra-edison";
 # network config file for inter-group connections
-   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison";
+   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/inter-edison";
 # routing protocol to be used 
    routing="non-minimal";
 }

src/network-workloads/conf/modelnet-test-dragonfly-theta.conf → src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf

@@ -35,20 +35,20 @@ PARAMS
 #bandwidth in GiB/s for local channels 
    local_bandwidth="5.25";
 # bandwidth in GiB/s for global channels 
-   global_bandwidth="4.7";
+   global_bandwidth="18.75";
 # bandwidth in GiB/s for compute node-router channels 
    cn_bandwidth="16.0";
 # ROSS message size 
-   message_size="584";
+   message_size="592";
 # number of compute nodes connected to router, dictated by dragonfly config
 # file
    num_cns_per_router="4";
 # number of global channels per router 
-   num_global_channels="12";
+   num_global_channels="10";
 # network config file for intra-group connections 
-   intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-theta";
+   intra-group-connections="/Users/mmubarak/Documents/software_development/test_versions/codes/scripts/gen-cray-topo/intra-theta";
 # network config file for inter-group connections
-   inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-theta";
+   inter-group-connections="/Users/mmubarak/Documents/software_development/test_versions/codes/scripts/gen-cray-topo/inter-theta";
 # routing protocol to be used 
    routing="prog-adaptive";
 }

src/network-workloads/model-net-mpi-replay.c

@@ -1640,7 +1640,6 @@ int main( int argc, char** argv )
 
    num_mpi_lps = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 0);
    num_net_lps = codes_mapping_get_lp_count("MODELNET_GRP", 1, MN_LP_NM, NULL, 0);
-   printf("\n num net lps %d ", num_net_lps);
     if (lp_io_dir[0]){
         do_lp_io = 1;
         /* initialize lp io */

src/network-workloads/model-net-synthetic-custom-dfly.c

@@ -126,7 +126,7 @@ const tw_lptype* svr_get_lp_type()
 
 static void svr_add_lp_type()
 {
-  lp_type_register("server", svr_get_lp_type());
+  lp_type_register("nw-lp", svr_get_lp_type());
 }
 
 static void issue_event(
@@ -382,7 +382,7 @@ int main(
         MPI_Finalize();
         return 0;
     }
-    num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "server",
+    num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "nw-lp",
             NULL, 1);
     configuration_get_value_int(&config, "PARAMS", "num_routers", NULL, &num_routers_per_grp);
     configuration_get_value_int(&config, "PARAMS", "num_groups", NULL, &num_groups);

src/networks/model-net/doc/README.dragonfly-custom.txt

@@ -60,12 +60,19 @@
   20 groups. Each group has 80 routers arranged in a 20x4 matrix]
 
   ./bin/model-net-synthetic-custom-dfly --sync=1 --
-  ../src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-custom.conf 
+  ../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf 
 
-  [With edison dragonfly network have 5,702 network nodes, 1440 routers and 15
+  [With theta dragonfly network having 3,456 compute nodes, 864 routers and 9
+  groups. Each group has 96 routers arranged in a 6x16 matrix]
+  
+  mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
+  ../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf
+
+  [With edison dragonfly network having 5,702 network nodes, 1440 routers and 15
   groups. Each group has 96 routers arranged in 6x16 matrix.]
+
   mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
-  ../src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-edison.conf 
+  ../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-edison.conf
 
 - Design Forward Network traces:
 
@@ -79,12 +86,19 @@
 
   [With Edison style dragonfly and AMG 1,728 application trace]
 
-  ./bin/model-net-mpi-replay --sync=1 --disable_compute=1
+  mpirun -np 4 ./bin/model-net-mpi-replay --sync=3 --disable_compute=1
   --workload_type="dumpi"
   --workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
   --num_net_traces=1728 --
-  ../src/network-workloads/conf/modelnet-test-dragonfly-edison.conf
+  ../src/network-workloads/dragonfly-custom/conf/modelnet-test-dragonfly-edison.conf
 
+  [With theta style dragonfly and AMG 1,728 application trace]
+
+  mpirun -np 4 ./bin/model-net-mpi-replay --sync=3 --disable_compute=1
+  --workload_type="dumpi"
+  --workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
+  --num_net_traces=1728 --
+  ../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-
 --------- Debugging Tips ------------
 - Set DUMP_CONNECTIONS debugging option to see the detailed local and global
   channel connectivity of routers in src/networks/model-net/dragonfly-custom.C

src/networks/model-net/dragonfly-custom.C

@@ -20,7 +20,7 @@
 #include <vector>
 #include <map>
 
-#define DUMP_CONNECTIONS 0
+#define DUMP_CONNECTIONS 1
 #define CREDIT_SIZE 8
 #define DFLY_HASH_TABLE_SIZE 262144
 
@@ -282,6 +282,11 @@ enum ROUTING_ALGO
     PROG_ADAPTIVE
 };
 
+enum LINK_TYPE
+{
+    GREEN,
+    BLACK,
+};
 struct router_state
 {
    unsigned int router_id;
@@ -690,9 +695,9 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
     }
 #endif
     if(!myRank) {
-        printf("\n Total nodes %d routers %d groups %d radix %d \n",
+        printf("\n Total nodes %d routers %d groups %d routers per group %d\n",
                 p->num_cn * p->total_routers, p->total_routers, p->num_groups,
-                p->radix);
+                p->num_routers);
     }
 
     p->cn_delay = bytes_to_ns(p->chunk_size, p->cn_bandwidth);