Commit 71544076 authored by Misbah Mubarak's avatar Misbah Mubarak

Merging custom dragonfly model with master, switching to libtools for build

parents b53ae378 716b39d5
Nikhil Jain (UIUC) Nikhil Jain, Abhinav Bhatele (LLNL)
- Improvements in credit-based flow control of CODES dragonfly and torus network models. - Improvements in credit-based flow control of CODES dragonfly and torus network models.
- Addition of direct scheme for setting up dragonfly network topology. - Addition of direct scheme for setting up dragonfly network topology.
- Network configuration setup for custom dragonfly model.
- Topology generations scripts for custom dragonfly model.
- Bug reporter for CODES network models. - Bug reporter for CODES network models.
- Fat tree network setup and adaptive routing.
Jens Domke (U. of Dresden)
- Static routing in fat tree network model.
Xu Yang (IIT) Xu Yang (IIT)
- Added support for running multiple application workloads with CODES MPI - Added support for running multiple application workloads with CODES MPI
......
...@@ -5,7 +5,7 @@ bin_PROGRAMS = ...@@ -5,7 +5,7 @@ bin_PROGRAMS =
bin_SCRIPTS = bin_SCRIPTS =
noinst_LIBRARIES = noinst_LIBRARIES =
noinst_PROGRAMS = noinst_PROGRAMS =
lib_LIBRARIES = lib_LTLIBRARIES =
noinst_HEADERS = noinst_HEADERS =
TESTS = TESTS =
check_PROGRAMS = check_PROGRAMS =
...@@ -24,13 +24,13 @@ EXTRA_DIST += \ ...@@ -24,13 +24,13 @@ EXTRA_DIST += \
misc/README misc/ptrn_loggp-2.4.6.patch CONTRIBUTORS.md \ misc/README misc/ptrn_loggp-2.4.6.patch CONTRIBUTORS.md \
README.md README.md
AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS} AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CFLAGS}
AM_CFLAGS = AM_CFLAGS =
AM_CXXFLAGS = $(AM_CFLAGS) AM_CXXFLAGS = $(AM_CFLAGS)
LDADD = $(lib_LIBRARIES) $(ROSS_LIBS) LDADD = $(lib_LTLIBRARIES) $(ROSS_LIBS)
include Make.rules include Make.rules
...@@ -41,18 +41,18 @@ include $(top_srcdir)/doc/Makefile.subdir ...@@ -41,18 +41,18 @@ include $(top_srcdir)/doc/Makefile.subdir
if USE_DARSHAN if USE_DARSHAN
AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1 AM_CPPFLAGS += ${DARSHAN_CFLAGS} -DUSE_DARSHAN=1
src_libcodes_a_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c src_libcodes_la_SOURCES += src/workload/methods/codes-darshan-io-wrkld.c
LDADD += ${DARSHAN_LIBS} LDADD += ${DARSHAN_LIBS}
TESTS += tests/workload/darshan-dump.sh TESTS += tests/workload/darshan-dump.sh
endif endif
if USE_RECORDER if USE_RECORDER
AM_CPPFLAGS += ${RECORDER_CPPFLAGS} AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
src_libcodes_a_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c src_libcodes_la_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
endif endif
if USE_DUMPI if USE_DUMPI
AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1 AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
src_libcodes_a_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
LDADD += ${DUMPI_LIBS} LDADD += ${DUMPI_LIBS}
endif endif
...@@ -64,6 +64,8 @@ typedef struct mn_stats mn_stats; ...@@ -64,6 +64,8 @@ typedef struct mn_stats mn_stats;
X(FATTREE, "modelnet_fattree", "fattree", &fattree_method)\ X(FATTREE, "modelnet_fattree", "fattree", &fattree_method)\
X(DRAGONFLY, "modelnet_dragonfly", "dragonfly", &dragonfly_method)\ X(DRAGONFLY, "modelnet_dragonfly", "dragonfly", &dragonfly_method)\
X(DRAGONFLY_ROUTER, "modelnet_dragonfly_router", "dragonfly_router", &dragonfly_router_method)\ X(DRAGONFLY_ROUTER, "modelnet_dragonfly_router", "dragonfly_router", &dragonfly_router_method)\
X(DRAGONFLY_CUSTOM, "modelnet_dragonfly_custom", "dragonfly_custom", &dragonfly_custom_method)\
X(DRAGONFLY_CUSTOM_ROUTER, "modelnet_dragonfly_custom_router", "dragonfly_custom_router", &dragonfly_custom_router_method)\
X(LOGGP, "modelnet_loggp", "loggp", &loggp_method)\ X(LOGGP, "modelnet_loggp", "loggp", &loggp_method)\
X(MAX_NETS, NULL, NULL, NULL) X(MAX_NETS, NULL, NULL, NULL)
......
/*
* Copyright (C) 2014 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef DRAGONFLY_CUSTOM_H
#define DRAGONFLY_CUSTOM_H
#ifdef __cplusplus
extern "C" {
#endif
#include <ross.h>
typedef struct terminal_custom_message terminal_custom_message;
/* this message is used for both dragonfly compute nodes and routers */
struct terminal_custom_message
{
/* magic number */
int magic;
/* flit travel start time*/
tw_stime travel_start_time;
/* packet ID of the flit */
unsigned long long packet_ID;
/* event type of the flit */
short type;
/* category: comes from codes */
char category[CATEGORY_NAME_MAX];
/* store category hash in the event */
uint32_t category_hash;
/* final destination LP ID, this comes from codes can be a server or any other LP type*/
tw_lpid final_dest_gid;
/*sending LP ID from CODES, can be a server or any other LP type */
tw_lpid sender_lp;
tw_lpid sender_mn_lp; // source modelnet id
/* destination terminal ID of the dragonfly */
tw_lpid dest_terminal_id;
/* source terminal ID of the dragonfly */
unsigned int src_terminal_id;
/* message originating router id. MM: Can we calculate it through
* sender_mn_lp??*/
unsigned int origin_router_id;
/* number of hops traversed by the packet */
short my_N_hop;
short my_l_hop, my_g_hop;
short saved_channel;
short saved_vc;
int next_stop;
short nonmin_done;
/* Intermediate LP ID from which this message is coming */
unsigned int intm_lp_id;
/* last hop of the message, can be a terminal, local router or global router */
short last_hop;
/* For routing */
int intm_rtr_id;
int saved_src_dest;
int saved_src_chan;
uint32_t chunk_id;
uint32_t packet_size;
uint32_t message_id;
uint32_t total_size;
int remote_event_size_bytes;
int local_event_size_bytes;
// For buffer message
short vc_index;
int output_chan;
model_net_event_return event_rc;
int is_pull;
uint32_t pull_size;
/* for reverse computation */
int path_type;
tw_stime saved_available_time;
tw_stime saved_avg_time;
tw_stime saved_rcv_time;
tw_stime saved_busy_time;
tw_stime saved_total_time;
tw_stime saved_sample_time;
tw_stime msg_start_time;
};
#ifdef __cplusplus
}
#endif
#endif /* end of include guard: DRAGONFLY_H */
/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
* End:
*
* vim: ft=c ts=8 sts=4 sw=4 expandtab
*/
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
#include <ross.h> #include <ross.h>
/* Global variable for modelnet output directory name */ /* Global variable for modelnet output directory name */
char *modelnet_stats_dir; extern char *modelnet_stats_dir;
typedef struct fattree_message fattree_message; typedef struct fattree_message fattree_message;
......
...@@ -11,11 +11,10 @@ ...@@ -11,11 +11,10 @@
extern "C" { extern "C" {
#endif #endif
typedef enum nodes_event_t nodes_event_t;
typedef struct nodes_message nodes_message; typedef struct nodes_message nodes_message;
/* event type of each torus message, can be packet generate, flit arrival, flit send or credit */ /* event type of each torus message, can be packet generate, flit arrival, flit send or credit */
enum nodes_event_t typedef enum nodes_event_t
{ {
GENERATE = 1, GENERATE = 1,
ARRIVAL, ARRIVAL,
...@@ -24,7 +23,7 @@ enum nodes_event_t ...@@ -24,7 +23,7 @@ enum nodes_event_t
T_COLLECTIVE_INIT, T_COLLECTIVE_INIT,
T_COLLECTIVE_FAN_IN, T_COLLECTIVE_FAN_IN,
T_COLLECTIVE_FAN_OUT T_COLLECTIVE_FAN_OUT
}; } nodes_event_t;
struct nodes_message struct nodes_message
{ {
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
AC_PREREQ([2.67]) AC_PREREQ([2.67])
AC_INIT([codes], [0.5.2], [http://trac.mcs.anl.gov/projects/codes/newticket],[],[http://www.mcs.anl.gov/projects/codes/]) AC_INIT([codes], [0.5.2], [http://trac.mcs.anl.gov/projects/codes/newticket],[],[http://www.mcs.anl.gov/projects/codes/])
LT_INIT
AC_CANONICAL_TARGET AC_CANONICAL_TARGET
AC_CANONICAL_SYSTEM AC_CANONICAL_SYSTEM
......
...@@ -319,7 +319,7 @@ transparent message passing support between LPs, allowing the underlying ...@@ -319,7 +319,7 @@ transparent message passing support between LPs, allowing the underlying
network LPs to do the work of routing while user LPs model their applications. network LPs to do the work of routing while user LPs model their applications.
It consists of a number of both simple and complex network models as well as It consists of a number of both simple and complex network models as well as
configuration utilities and communication APIs. A somewhat stale overview is configuration utilities and communication APIs. A somewhat stale overview is
also given at src/models/networks/model-net/doc/README. also given at src/networks/model-net/doc/README.
= Components of model-net = Components of model-net
...@@ -407,8 +407,8 @@ for the latency and bandwidth costs: "net_latency_ns_file" and ...@@ -407,8 +407,8 @@ for the latency and bandwidth costs: "net_latency_ns_file" and
"net_bw_mbps_file". "net_bw_mbps_file".
More details about the models can be found at More details about the models can be found at
src/models/networks/model-net/doc/README.simplenet.txt and src/networks/model-net/doc/README.simplenet.txt and
src/models/networks/model-net/doc/README.simplep2p.txt, respectively. src/networks/model-net/doc/README.simplep2p.txt, respectively.
== LogGP == LogGP
...@@ -422,7 +422,7 @@ The only configuration entry the LogGP model requires is ...@@ -422,7 +422,7 @@ The only configuration entry the LogGP model requires is
configuration file. configuration file.
For more details on gathering parameters for the LogGP model, as well as it's For more details on gathering parameters for the LogGP model, as well as it's
usage and caveats, see the document src/models/model-net/doc/README.loggp.txt. usage and caveats, see the document src/model-net/doc/README.loggp.txt.
== Torus == Torus
...@@ -435,7 +435,7 @@ performed in: ...@@ -435,7 +435,7 @@ performed in:
(PADS), 2014. (PADS), 2014.
The configuration and model setup can be found at: The configuration and model setup can be found at:
src/models/model-net/doc/README.torus.txt src/model-net/doc/README.torus.txt
== Dragonfly == Dragonfly
...@@ -443,7 +443,7 @@ The dragonfly model (model-net LP name: "dragonfly") is a network ...@@ -443,7 +443,7 @@ The dragonfly model (model-net LP name: "dragonfly") is a network
topology that utilizes the concept of virtual routers to produce systems with topology that utilizes the concept of virtual routers to produce systems with
very high virtual radix out of network components with a lower radix. The very high virtual radix out of network components with a lower radix. The
topology itself and the simulation model are both described in topology itself and the simulation model are both described in
src/models/networks/model-net/doc/README.dragonfly.txt. src/networks/model-net/doc/README.dragonfly.txt.
cite). cite).
The configuration parameters are a little trickier here, as additional LPs The configuration parameters are a little trickier here, as additional LPs
...@@ -453,7 +453,9 @@ represents a physical router. At least one "dragonfly_router" LP must be ...@@ -453,7 +453,9 @@ represents a physical router. At least one "dragonfly_router" LP must be
present in every LP group with a "modelnet_dragonfly" LP. present in every LP group with a "modelnet_dragonfly" LP.
Further configuration and model setup can be found at Further configuration and model setup can be found at
src/models/model-net/doc/README.dragonfly.txt. src/model-net/doc/README.dragonfly.txt.
A newer version of dragonfly model that supports a Cray style network is also
available. Instructions can be found at src/model-net/doc/README.dragonfly-custom.txt.
= CODES example model = CODES example model
......
//////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
// Produced at the Lawrence Livermore National Laboratory.
//
// Written by:
// Nikhil Jain <nikhil.jain@acm.org>
// Abhinav Bhatele <bhatele@llnl.gov>
// Peer-Timo Bremer <ptbremer@llnl.gov>
//
// LLNL-CODE-678961. All rights reserved.
//
// This file is part of Damselfly. For details, see:
// https://github.com/LLNL/damselfly
// Please also read the LICENSE file for our notice and the LGPL.
//////////////////////////////////////////////////////////////////////////////
#include "stdio.h"
#include "stdlib.h"
//Usage ./binary num_groups num_rows num_columns intra_file inter_file
int main(int argc, char **argv) {
int g = atoi(argv[1]);
int r = atoi(argv[2]);
int c = atoi(argv[3]);
FILE *intra = fopen(argv[4], "w+");
FILE *inter = fopen(argv[5], "w+");
int router = 0;
int dest = 0;
int num_globs = 4;
int green = 0;
int black = 1;
int blue = 2;
printf("\n Num groups %d num_global_chans %d num_rows %d num_cols %d ", g, num_globs, r, c);
for(int groups = 0; groups < g; groups++)
{
/* First connect the router to other routers in the same row */
for(int rows = 0; rows < r; rows++) {
int offset = c * rows;
for(int out_col = 0; out_col < c; out_col++)
{
if(groups == 0)
{
/* Do it for group 0 only */
for(int cols = 0; cols < c; cols++) {
dest = offset + cols;
if((router % (c * r)) != dest)
{
fwrite(&router, sizeof(int), 1, intra);
fwrite(&dest, sizeof(int), 1, intra);
fwrite(&green, sizeof(int), 1, intra);
printf("\n INTRA Same row %d %d ", router, dest);
}
}
for(int r_up = 0; r_up < r; r_up++)
{
dest = (c * r_up) + (router % c);
if((router % (c * r)) != dest)
{
fwrite(&router, sizeof(int), 1, intra);
fwrite(&dest, sizeof(int), 1, intra);
fwrite(&black, sizeof(int), 1, intra);
printf("\n INTRA Same col %d %d ", router, dest);
}
}
} // end if
// Now setup global connections
//
int myOff = router % (r * c);
int numLink = g / (r*c);
if(g % (r*c) != 0) {
if((router % (r*c)) < (g % (r*c))) {
numLink++;
}
}
int myG = router / (r * c);
for(int blues = 0; blues < numLink; blues++) {
int dest = (blues * r * c) + myOff;
if(dest != myG) {
dest = (dest * r * c ) + (myG % (r * c));
for(int pair = 0; pair < 2; pair++)
{
fwrite(&router, sizeof(int), 1, inter);
fwrite(&dest, sizeof(int), 1, inter);
printf("INTER %d %d %d \n", router, dest, blue);
}
}
}
router++;
}
}
}
fclose(intra);
fclose(inter);
}
** Generating inter and intra group files for Edison and Theta Interconnects **:
- Edison network config files:
python edison.py link-edison.txt intra-edison inter-edison
[intra-edison and inter-edison are the intra-group and inter-group network
configuration files required by the simulation. The python script
translates Edison's network configuration into a file format that can be fed
into the simulation.]
** Generating customizable dragonfly interconnects **:
mpicc connections_general.c -o connections_general
./connections_general g r c intra-file inter-file
--> g: number of groups in the network
--> r: number of router rows within a group
--> c: number of router columns within a group
--> intra-file: output files for intra-group connections
--> inter-file: output file for inter-group connections
- The scripts and code for translating existing topologies and generating
cray-style dragonfly topologies have been contributed by Nikhil Jain, Abhinav
Bhatele and Peer-Timo Breemer from LLNL.
- For details on cray XC dragonfly network topology, see the following paper:
@inproceedings{faanes2012cray,
title={Cray cascade: a scalable HPC system based on a Dragonfly network},
author={Faanes, Greg and Bataineh, Abdulla and Roweth, Duncan and Froese,
Edwin and Alverson, Bob and Johnson, Tim and Kopnick, Joe and Higgins, Mike
and Reinhard, James and others},
booktitle={Proceedings of the International Conference on High
Performance Computing, Networking, Storage and Analysis},
pages={103},
year={2012},
organization={IEEE Computer Society Press}
}
//////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
// Produced at the Lawrence Livermore National Laboratory.
//
// Written by:
// Nikhil Jain <nikhil.jain@acm.org>
// Abhinav Bhatele <bhatele@llnl.gov>
// Peer-Timo Bremer <ptbremer@llnl.gov>
//
// LLNL-CODE-678961. All rights reserved.
//
// This file is part of Damselfly. For details, see:
// https://github.com/scalability-llnl/damselfly
// Please also read the LICENSE file for our notice and the LGPL.
//////////////////////////////////////////////////////////////////////////////
#include "stdio.h"
#include "stdlib.h"
//Usage ./binary num_groups num_rows num_columns intra_file inter_file
int main(int argc, char **argv) {
if(argc < 3) {
printf("Correct usage: %s <num_g> <num_rows> <num_cols> <intra_file> <inter_file>", argv[0]);
exit(0);
}
int g = atoi(argv[1]);
int r = atoi(argv[2]);
int c = atoi(argv[3]);
FILE *intra = fopen(argv[4], "wb");
FILE *inter = fopen(argv[5], "wb");
int router = 0;
int green = 0, black = 1;
int groups = 0;
for(int rows = 0; rows < r; rows++) {
for(int cols = 0; cols < c; cols++) {
for(int cols1 = 0; cols1 < c; cols1++) {
if(cols1 != cols) {
int dest = (rows * c) + cols1;
fwrite(&router, sizeof(int), 1, intra);
fwrite(&dest, sizeof(int), 1, intra);
fwrite(&green, sizeof(int), 1, intra);
// printf("INTRA %d %d %d\n", router, dest, green);
}
}
for(int rows1 = 0; rows1 < r; rows1++) {
if(rows1 != rows) {
int dest = (rows1 * c) + cols;
for(int link = 0; link < 3; link++) {
fwrite(&router, sizeof(int), 1, intra);
fwrite(&dest, sizeof(int), 1, intra);
fwrite(&black, sizeof(int), 1, intra);
// printf("INTRA %d %d %d\n", router, dest, black);
}
}
}
router++;
}
}
for(int srcg = 0; srcg < g; srcg++) {
for(int dstg = 0; dstg < g; dstg++) {
if(srcg != dstg) {
int nsrcg = srcg;
int ndstg = dstg;
if(srcg > dstg) {
nsrcg--;
} else {
ndstg--;
}
int gsize = 2, gs = 16;
for(int row = 0; row < r; row++) {
int srcrB = srcg * r * c + row * c, srcr;
int dstrB = dstg * r * c + row * c, dstr;
int srcB = (ndstg % (gs/2)) * 2;
int dstB = (nsrcg % (gs/2)) * 2;
srcr = srcrB + srcB;
dstr = dstrB + dstB;
for(int r = 0; r < 2; r++) {
for(int block = 0; block < gsize; block++) {
fwrite(&srcr, sizeof(int), 1, inter);
fwrite(&dstr, sizeof(int), 1, inter);
printf("INTER %d %d\n", srcr, dstr);
}
srcr++;
dstr++;
}
}
}
}
}
fclose(intra);
fclose(inter);
}
#!/usr/bin/env python
##############################################################################
# Copyright (c) 2014, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# Written by:
# Nikhil Jain <nikhil.jain@acm.org>
# Abhinav Bhatele <bhatele@llnl.gov>
# Peer-Timo Bremer <ptbremer@llnl.gov>
#
# LLNL-CODE-678961. All rights reserved.
#
# This file is part of Damselfly. For details, see:
# https://github.com/LLNL/damselfly
# Please also read the LICENSE file for our notice and the LGPL.
##############################################################################
import sys
import re
import numpy as np
import struct
filename = sys.argv[1]
intracon = open(sys.argv[2], "wb")
intercon = open(sys.argv[3], "wb")
def router(group, row, col):
return group*96 + row*16 + col
numblack = np.zeros((1440,1440), dtype=np.int)
numblue = np.zeros((1440,1440), dtype=np.int)
with open(filename) as ofile:
matches = re.findall('c\d-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\).(\w+).->.c\d-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\)', ofile.read(), re.MULTILINE)
for match in matches:
srcgrp = int(match[0])
if(srcgrp > 12):
srcgrp = srcgrp - 1
srcrow = int(match[1])
srccol = int(match[2])
srcrouter = router(srcgrp, srcrow, srccol)
color = match[3]
dstgrp = int(match[4])
if(dstgrp > 12):
dstgrp = dstgrp - 1
dstrow = int(match[5])
dstcol = int(match[6])
dstrouter = router(dstgrp, dstrow, dstcol)
# count number of black and blue links per router pair
if color == 'black':
numblack[srcrouter][dstrouter] += 1
if color == 'blue':
numblue[srcrouter][dstrouter] += 1
if srcgrp == 0:
if color == 'blue':
# write to inter-con file
intercon.write(struct.pack('2i', srcrouter, dstrouter))
print 'BLUE', srcrouter, dstrouter
else:
# write to intra-con file
if color == 'green':
intracon.write(struct.pack('3i', srcrouter, dstrouter, 0))
print 'GREEN', srcrouter, dstrouter, 0
elif numblack[srcrouter][dstrouter] < 4:
intracon.write(struct.pack('3i', srcrouter, dstrouter, 1))
print 'BLACK', srcrouter, dstrouter, 1
else:
if color == 'blue':
# only write the inter-con file
intercon.write(struct.pack('2i', srcrouter, dstrouter))
print 'BLUE', srcrouter, dstrouter
for i in range(0, 1440):
for j in range(0, 1440):
if(numblack[i][j] != 0):
print numblack[i][j],
print "\n"
for i in range(0, 1440):
for j in range(0, 1440):
if(numblue[i][j] != 0):
print numblue[i][j],
intracon.close()
intercon.close()
This diff is collapsed.
//////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2014, Lawrence Livermore National Security, LLC.
// Produced at the Lawrence Livermore National Laboratory.
//
// Written by:
// Nikhil Jain <nikhil.jain@acm.org>
// Abhinav Bhatele <bhatele@llnl.gov>
// Peer-Timo Bremer <ptbremer@llnl.gov>
//
// LLNL-CODE-678961. All rights reserved.
//
// This file is part of Damselfly. For details, see:
// https://github.com/LLNL/damselfly
// Please also read the LICENSE file for our notice and the LGPL.
//////////////////////////////////////////////////////////////////////////////
#include "stdio.h"
#include "stdlib.h"
//Usage ./binary num_groups num_rows num_columns intra_file inter_file
int main(int argc, char **argv) {
int g = atoi(argv[1]);
int r = atoi(argv[2]);
int c = 1;
FILE *intra = fopen(argv[3], "wb");
FILE *inter = fopen(argv[4], "wb");
int router = 0;
int green = 0;
for(int groups = 0; groups < g; groups++) {
for(int rows = 0; rows < r; rows++) {
if(groups == 0) {
for(int rows1 = 0; rows1 < r; rows1++) {
if(rows1 != rows) {
int dest = rows1;
fwrite(&router, sizeof(int), 1, intra);
fwrite(&dest, sizeof(int), 1, intra);
fwrite(&green, sizeof(int), 1, intra);
printf("INTRA %d %d %d\n", router, dest, green);
}
}
}
int myOff = router % (r * c);
int numLink = g / (r*c);