Merging custom dragonfly model with master, switching to libtools for build

CONTRIBUTORS.md

-Nikhil Jain (UIUC)
+Nikhil Jain, Abhinav Bhatele (LLNL)
     - Improvements in credit-based flow control of CODES dragonfly and torus network models.
     - Addition of direct scheme for setting up dragonfly network topology.
+    - Network configuration setup for custom dragonfly model.
+    - Topology generations scripts for custom dragonfly model.
     - Bug reporter for CODES network models.
+    - Fat tree network setup and adaptive routing.
+
+Jens Domke (U. of Dresden)
+    - Static routing in fat tree network model.
 
 Xu Yang (IIT)
     - Added support for running multiple application workloads with CODES MPI

Makefile.am

@@ -5,7 +5,7 @@ bin_PROGRAMS =
 bin_SCRIPTS =
 noinst_LIBRARIES =
 noinst_PROGRAMS =
-lib_LIBRARIES =
+lib_LTLIBRARIES =
 noinst_HEADERS =
 TESTS =
 check_PROGRAMS =
@@ -24,13 +24,13 @@ EXTRA_DIST += \
   misc/README misc/ptrn_loggp-2.4.6.patch CONTRIBUTORS.md \
   README.md
 
-AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS}
+AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS} 
 
 AM_CFLAGS =
 
 AM_CXXFLAGS = $(AM_CFLAGS)
 
-LDADD = $(lib_LIBRARIES) $(ROSS_LIBS)
+LDADD = $(lib_LTLIBRARIES) $(ROSS_LIBS)
 
 include Make.rules
 
@@ -41,18 +41,18 @@ include $(top_srcdir)/doc/Makefile.subdir
 
 if USE_DARSHAN
 AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1
-src_libcodes_a_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c
+src_libcodes_la_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c
 LDADD += ${DARSHAN_LIBS}
 TESTS += tests/workload/darshan-dump.sh
 endif
 
 if USE_RECORDER
 AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
-src_libcodes_a_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
+src_libcodes_la_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
 endif
 
 if USE_DUMPI
 AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
-src_libcodes_a_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
+src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
 LDADD += ${DUMPI_LIBS}
 endif

codes/model-net.h

@@ -64,6 +64,8 @@ typedef struct mn_stats mn_stats;
     X(FATTREE,   "modelnet_fattree",   "fattree",   &fattree_method)\
     X(DRAGONFLY, "modelnet_dragonfly", "dragonfly", &dragonfly_method)\
     X(DRAGONFLY_ROUTER, "modelnet_dragonfly_router", "dragonfly_router", &dragonfly_router_method)\
+    X(DRAGONFLY_CUSTOM, "modelnet_dragonfly_custom", "dragonfly_custom", &dragonfly_custom_method)\
+    X(DRAGONFLY_CUSTOM_ROUTER, "modelnet_dragonfly_custom_router", "dragonfly_custom_router", &dragonfly_custom_router_method)\
     X(LOGGP,     "modelnet_loggp",     "loggp",     &loggp_method)\
     X(MAX_NETS,  NULL,                 NULL,        NULL)
 

codes/net/dragonfly-custom.h

+/*
+ * Copyright (C) 2014 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef DRAGONFLY_CUSTOM_H
+#define DRAGONFLY_CUSTOM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <ross.h>
+
+typedef struct terminal_custom_message terminal_custom_message;
+
+/* this message is used for both dragonfly compute nodes and routers */
+struct terminal_custom_message
+{
+  /* magic number */
+  int magic;
+  /* flit travel start time*/
+  tw_stime travel_start_time;
+ /* packet ID of the flit  */
+  unsigned long long packet_ID;
+  /* event type of the flit */
+  short  type;
+  /* category: comes from codes */
+  char category[CATEGORY_NAME_MAX];
+  /* store category hash in the event */
+  uint32_t category_hash;
+  /* final destination LP ID, this comes from codes can be a server or any other LP type*/
+  tw_lpid final_dest_gid;
+  /*sending LP ID from CODES, can be a server or any other LP type */
+  tw_lpid sender_lp;
+  tw_lpid sender_mn_lp; // source modelnet id
+ /* destination terminal ID of the dragonfly */
+  tw_lpid dest_terminal_id;
+  /* source terminal ID of the dragonfly */
+  unsigned int src_terminal_id;
+  /* message originating router id. MM: Can we calculate it through
+   * sender_mn_lp??*/
+  unsigned int origin_router_id;
+
+  /* number of hops traversed by the packet */
+  short my_N_hop;
+  short my_l_hop, my_g_hop;
+  short saved_channel;
+  short saved_vc;
+
+  int next_stop;
+
+  short nonmin_done;
+  /* Intermediate LP ID from which this message is coming */
+  unsigned int intm_lp_id;
+  /* last hop of the message, can be a terminal, local router or global router */
+  short last_hop;
+   /* For routing */
+  int intm_rtr_id;
+  int saved_src_dest;
+  int saved_src_chan;
+
+   uint32_t chunk_id;
+   uint32_t packet_size;
+   uint32_t message_id;
+   uint32_t total_size;
+
+   int remote_event_size_bytes;
+   int local_event_size_bytes;
+
+  // For buffer message
+   short vc_index;
+   int output_chan;
+   model_net_event_return event_rc;
+   int is_pull;
+   uint32_t pull_size;
+
+   /* for reverse computation */   
+   int path_type;
+   tw_stime saved_available_time;
+   tw_stime saved_avg_time;
+   tw_stime saved_rcv_time;
+   tw_stime saved_busy_time; 
+   tw_stime saved_total_time;
+   tw_stime saved_sample_time;
+   tw_stime msg_start_time;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* end of include guard: DRAGONFLY_H */
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

codes/net/fattree.h

@@ -4,7 +4,7 @@
 #include <ross.h>
 
 /* Global variable for modelnet output directory name */
-char *modelnet_stats_dir;
+extern char *modelnet_stats_dir;
 
 typedef struct fattree_message fattree_message;
 

codes/net/torus.h

@@ -11,11 +11,10 @@
 extern "C" {
 #endif
 
-typedef enum nodes_event_t nodes_event_t;
 typedef struct nodes_message nodes_message;
 
 /* event type of each torus message, can be packet generate, flit arrival, flit send or credit */
-enum nodes_event_t
+typedef enum nodes_event_t
 {
   GENERATE = 1,
   ARRIVAL, 
@@ -24,7 +23,7 @@ enum nodes_event_t
   T_COLLECTIVE_INIT,
   T_COLLECTIVE_FAN_IN,
   T_COLLECTIVE_FAN_OUT  
-};
+} nodes_event_t;
 
 struct nodes_message
 {

configure.ac

@@ -3,6 +3,7 @@
 
 AC_PREREQ([2.67])
 AC_INIT([codes], [0.5.2], [http://trac.mcs.anl.gov/projects/codes/newticket],[],[http://www.mcs.anl.gov/projects/codes/])
+LT_INIT
 
 AC_CANONICAL_TARGET
 AC_CANONICAL_SYSTEM

doc/GETTING_STARTED

@@ -319,7 +319,7 @@ transparent message passing support between LPs, allowing the underlying
 network LPs to do the work of routing while user LPs model their applications.
 It consists of a number of both simple and complex network models as well as
 configuration utilities and communication APIs. A somewhat stale overview is
-also given at src/models/networks/model-net/doc/README.
+also given at src/networks/model-net/doc/README.
 
 = Components of model-net
 
@@ -407,8 +407,8 @@ for the latency and bandwidth costs: "net_latency_ns_file" and
 "net_bw_mbps_file".
 
 More details about the models can be found at
-src/models/networks/model-net/doc/README.simplenet.txt and
-src/models/networks/model-net/doc/README.simplep2p.txt, respectively.
+src/networks/model-net/doc/README.simplenet.txt and
+src/networks/model-net/doc/README.simplep2p.txt, respectively.
 
 == LogGP
 
@@ -422,7 +422,7 @@ The only configuration entry the LogGP model requires is
 configuration file.
 
 For more details on gathering parameters for the LogGP model, as well as it's
-usage and caveats, see the document src/models/model-net/doc/README.loggp.txt. 
+usage and caveats, see the document src/model-net/doc/README.loggp.txt. 
 
 == Torus
 
@@ -435,7 +435,7 @@ performed in:
     (PADS), 2014.
 
 The configuration and model setup can be found at:
-src/models/model-net/doc/README.torus.txt
+src/model-net/doc/README.torus.txt
 
 == Dragonfly
 
@@ -443,7 +443,7 @@ The dragonfly model (model-net LP name: "dragonfly") is a network
 topology that utilizes the concept of virtual routers to produce systems with
 very high virtual radix out of network components with a lower radix. The
 topology itself and the simulation model are both described in
-src/models/networks/model-net/doc/README.dragonfly.txt.
+src/networks/model-net/doc/README.dragonfly.txt.
 cite).
 
 The configuration parameters are a little trickier here, as additional LPs
@@ -453,7 +453,9 @@ represents a physical router. At least one "dragonfly_router" LP must be
 present in every LP group with a "modelnet_dragonfly" LP. 
 
 Further configuration and model setup can be found at
-src/models/model-net/doc/README.dragonfly.txt.
+src/model-net/doc/README.dragonfly.txt.
+A newer version of dragonfly model that supports a Cray style network is also
+available. Instructions can be found at src/model-net/doc/README.dragonfly-custom.txt.
 
 = CODES example model
 

scripts/cray-topo.c

+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
+// Produced at the Lawrence Livermore National Laboratory.
+//
+// Written by:
+//     Nikhil Jain <nikhil.jain@acm.org>
+//     Abhinav Bhatele <bhatele@llnl.gov>
+//     Peer-Timo Bremer <ptbremer@llnl.gov>
+//
+// LLNL-CODE-678961. All rights reserved.
+//
+// This file is part of Damselfly. For details, see:
+// https://github.com/LLNL/damselfly
+// Please also read the LICENSE file for our notice and the LGPL.
+//////////////////////////////////////////////////////////////////////////////
+
+#include "stdio.h"
+#include "stdlib.h"
+
+//Usage ./binary num_groups num_rows num_columns intra_file inter_file
+
+int main(int argc, char **argv) {
+  int g = atoi(argv[1]);
+  int r = atoi(argv[2]);
+  int c = atoi(argv[3]);
+
+  FILE *intra = fopen(argv[4], "w+");
+  FILE *inter = fopen(argv[5], "w+");
+ 
+  int router = 0;
+  int dest = 0;
+  int num_globs = 4;
+
+  int green = 0;
+  int black = 1;
+  int blue = 2;
+
+  printf("\n Num groups %d num_global_chans %d num_rows %d num_cols %d ", g, num_globs, r, c);
+
+  for(int groups = 0; groups < g; groups++)
+  {
+      /* First connect the router to other routers in the same row */
+      for(int rows = 0; rows < r; rows++) {
+        int offset = c * rows;
+        for(int out_col = 0; out_col < c; out_col++)
+        {
+            if(groups == 0)
+            {
+                /* Do it for group 0 only */
+                for(int cols = 0; cols < c; cols++) {
+                    dest = offset + cols;
+                    
+                    if((router % (c * r)) != dest)
+                    {
+                       fwrite(&router, sizeof(int), 1, intra);
+                       fwrite(&dest, sizeof(int), 1, intra);
+                       fwrite(&green, sizeof(int), 1, intra);
+                       printf("\n INTRA Same row %d %d ", router, dest);
+                    }
+               }
+                for(int r_up = 0; r_up < r; r_up++)
+                {
+                    dest = (c * r_up) + (router % c);
+                    if((router % (c * r)) != dest)
+                    {
+                       fwrite(&router, sizeof(int), 1, intra);
+                       fwrite(&dest, sizeof(int), 1, intra);
+                       fwrite(&black, sizeof(int), 1, intra);
+                       printf("\n INTRA Same col %d %d ", router, dest);
+                    }
+                }
+           } // end if 
+     
+       // Now setup global connections
+       //
+      int myOff = router % (r * c);
+      int numLink = g / (r*c);
+      if(g % (r*c) != 0) {
+        if((router % (r*c)) < (g % (r*c))) {
+          numLink++;
+        }
+      }
+      int myG = router / (r * c);
+      for(int blues = 0; blues < numLink; blues++) {
+        int dest = (blues * r * c) + myOff;
+        if(dest != myG) {
+          dest = (dest * r * c ) + (myG % (r * c));
+          for(int pair = 0; pair < 2; pair++)
+          {
+            fwrite(&router, sizeof(int), 1, inter);
+            fwrite(&dest, sizeof(int), 1, inter);
+            printf("INTER %d %d %d \n", router, dest, blue);
+          }
+        }
+      }
+        router++;
+    }
+  }
+  }
+  fclose(intra);
+  fclose(inter);
+}

scripts/gen-cray-topo/README.txt

+** Generating inter and intra group files for Edison and Theta Interconnects **:
+
+- Edison network config files:
+    python edison.py link-edison.txt intra-edison inter-edison
+    
+    [intra-edison and inter-edison are the intra-group and inter-group network
+    configuration files required by the simulation. The python script
+    translates Edison's network configuration into a file format that can be fed
+    into the simulation.]
+
+ ** Generating customizable dragonfly interconnects **:
+
+mpicc connections_general.c -o connections_general
+
+./connections_general g r c intra-file inter-file
+
+--> g: number of groups in the network
+--> r: number of router rows within a group
+--> c: number of router columns within a group
+--> intra-file: output files for intra-group connections
+--> inter-file: output file for inter-group connections
+
+- The scripts and code for translating existing topologies and generating
+  cray-style dragonfly topologies have been contributed by Nikhil Jain, Abhinav
+  Bhatele and Peer-Timo Breemer from LLNL.
+
+- For details on cray XC dragonfly network topology, see the following paper:
+
+@inproceedings{faanes2012cray,
+    title={Cray cascade: a scalable HPC system based on a Dragonfly network},
+    author={Faanes, Greg and Bataineh, Abdulla and Roweth, Duncan and Froese,
+    Edwin and Alverson, Bob and Johnson, Tim and Kopnick, Joe and Higgins, Mike
+    and Reinhard, James and others},
+    booktitle={Proceedings of the International Conference on High
+    Performance Computing, Networking, Storage and Analysis},
+    pages={103},
+    year={2012},
+    organization={IEEE Computer Society Press}
+}

scripts/gen-cray-topo/connections_general.c

+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
+// Produced at the Lawrence Livermore National Laboratory.
+//
+// Written by:
+//     Nikhil Jain <nikhil.jain@acm.org>
+//     Abhinav Bhatele <bhatele@llnl.gov>
+//     Peer-Timo Bremer <ptbremer@llnl.gov>
+//
+// LLNL-CODE-678961. All rights reserved.
+//
+// This file is part of Damselfly. For details, see:
+// https://github.com/scalability-llnl/damselfly
+// Please also read the LICENSE file for our notice and the LGPL.
+//////////////////////////////////////////////////////////////////////////////
+
+#include "stdio.h"
+#include "stdlib.h"
+
+//Usage ./binary num_groups num_rows num_columns intra_file inter_file
+
+int main(int argc, char **argv) {
+  if(argc < 3) {
+    printf("Correct usage: %s <num_g> <num_rows> <num_cols> <intra_file> <inter_file>", argv[0]);
+    exit(0);
+  }
+
+  int g = atoi(argv[1]);
+  int r = atoi(argv[2]);
+  int c = atoi(argv[3]);
+
+  FILE *intra = fopen(argv[4], "wb");
+  FILE *inter = fopen(argv[5], "wb");
+
+  int router = 0;
+  int green = 0, black = 1;
+  int groups = 0;
+  for(int rows = 0; rows < r; rows++) {
+    for(int cols = 0; cols < c; cols++) {
+      for(int cols1 = 0; cols1 < c; cols1++) {
+        if(cols1 != cols) {
+          int dest = (rows * c) + cols1;
+          fwrite(&router, sizeof(int), 1, intra);
+          fwrite(&dest, sizeof(int), 1, intra);
+          fwrite(&green, sizeof(int), 1, intra);
+          // printf("INTRA %d %d %d\n", router, dest, green);
+        }
+      }
+      for(int rows1 = 0; rows1 < r; rows1++) {
+        if(rows1 != rows) {
+          int dest = (rows1 * c) + cols;
+          for(int link = 0; link < 3; link++) {
+            fwrite(&router, sizeof(int), 1, intra);
+            fwrite(&dest, sizeof(int), 1, intra);
+            fwrite(&black, sizeof(int), 1, intra);
+            // printf("INTRA %d %d %d\n", router, dest, black);
+          }
+        }
+      }
+      router++;
+    }
+  }
+
+  for(int srcg = 0; srcg < g; srcg++) {
+    for(int dstg = 0; dstg < g; dstg++) {
+      if(srcg != dstg) {
+        int nsrcg = srcg;
+        int ndstg = dstg;
+        if(srcg > dstg) {
+          nsrcg--;
+        } else {
+          ndstg--;
+        }
+        int gsize = 2, gs = 16;
+        for(int row = 0; row < r; row++) {
+          int srcrB = srcg * r * c + row * c, srcr;
+          int dstrB = dstg * r * c + row * c, dstr;
+          int srcB = (ndstg % (gs/2)) * 2;
+          int dstB = (nsrcg % (gs/2)) * 2;
+          srcr = srcrB + srcB;
+          dstr = dstrB + dstB;
+          for(int r = 0; r < 2; r++) {
+            for(int block = 0; block < gsize; block++) {
+              fwrite(&srcr, sizeof(int), 1, inter);
+              fwrite(&dstr, sizeof(int), 1, inter);
+              printf("INTER %d %d\n", srcr, dstr);
+            }
+            srcr++;
+            dstr++;
+          }
+        }
+      }
+    }
+  }
+
+  fclose(intra);
+  fclose(inter);
+}

scripts/gen-cray-topo/edison.py

+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2014, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# Written by:
+#     Nikhil Jain <nikhil.jain@acm.org>
+#     Abhinav Bhatele <bhatele@llnl.gov>
+#     Peer-Timo Bremer <ptbremer@llnl.gov>
+#
+# LLNL-CODE-678961. All rights reserved.
+#
+# This file is part of Damselfly. For details, see:
+# https://github.com/LLNL/damselfly
+# Please also read the LICENSE file for our notice and the LGPL.
+##############################################################################
+
+import sys
+import re
+import numpy as np
+import struct
+
+filename = sys.argv[1]
+intracon = open(sys.argv[2], "wb")
+intercon = open(sys.argv[3], "wb")
+
+def router(group, row, col):
+    return group*96 + row*16 + col
+
+numblack = np.zeros((1440,1440), dtype=np.int)
+numblue = np.zeros((1440,1440), dtype=np.int)
+
+with open(filename) as ofile:
+    matches = re.findall('c\d-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\).(\w+).->.c\d-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\)', ofile.read(), re.MULTILINE)
+
+for match in matches:
+    srcgrp = int(match[0])
+    if(srcgrp > 12):
+	srcgrp = srcgrp - 1
+    srcrow = int(match[1])
+    srccol = int(match[2])
+    srcrouter = router(srcgrp, srcrow, srccol)
+
+    color = match[3]
+
+    dstgrp = int(match[4])
+    if(dstgrp > 12):
+	dstgrp = dstgrp - 1
+    dstrow = int(match[5])
+    dstcol = int(match[6])
+    dstrouter = router(dstgrp, dstrow, dstcol)
+
+    # count number of black and blue links per router pair
+    if color == 'black':
+	numblack[srcrouter][dstrouter] += 1
+    if color == 'blue':
+	numblue[srcrouter][dstrouter] += 1
+
+    if srcgrp == 0:
+	if color == 'blue':
+	    # write to inter-con file
+	    intercon.write(struct.pack('2i', srcrouter, dstrouter))
+	    print 'BLUE', srcrouter, dstrouter
+	else:
+	    # write to intra-con file
+	    if color == 'green':
+		intracon.write(struct.pack('3i', srcrouter, dstrouter, 0))
+		print 'GREEN', srcrouter, dstrouter, 0
+	    elif numblack[srcrouter][dstrouter] < 4:
+		intracon.write(struct.pack('3i', srcrouter, dstrouter, 1))
+		print 'BLACK', srcrouter, dstrouter, 1
+    else:
+	if color == 'blue':
+	    # only write the inter-con file
+	    intercon.write(struct.pack('2i', srcrouter, dstrouter))
+	    print 'BLUE', srcrouter, dstrouter
+
+for i in range(0, 1440):
+    for j in range(0, 1440):
+	if(numblack[i][j] != 0):
+	    print numblack[i][j],
+
+print "\n"
+for i in range(0, 1440):
+    for j in range(0, 1440):
+	if(numblue[i][j] != 0):
+	    print numblue[i][j],
+
+intracon.close()
+intercon.close()
+

scripts/simple_con.c

+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2014, Lawrence Livermore National Security, LLC.