Commit 29bd0c1e authored by Misbah Mubarak's avatar Misbah Mubarak

Adding theta network configuration

parent abcccfe0
......@@ -8,6 +8,13 @@
translates Edison's network configuration into a file format that can be fed
into the simulation.]
- Theta network config files:
python theta.py theta.interconnect intra-theta inter-theta
[intra-theta and inter-theta are the intra and inter-group config files for
dragonfly. ]
** Generating customizable dragonfly interconnects **:
mpicc connections_general.c -o connections_general
......
This diff is collapsed.
#!/usr/bin/env python
##############################################################################
# Copyright (c) 2014, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# Written by:
# Nikhil Jain <nikhil.jain@acm.org>
# Abhinav Bhatele <bhatele@llnl.gov>
# Peer-Timo Bremer <ptbremer@llnl.gov>
#
# LLNL-CODE-678961. All rights reserved.
#
# This file is part of Damselfly. For details, see:
# https://github.com/LLNL/damselfly
# Please also read the LICENSE file for our notice and the LGPL.
##############################################################################
# Modified by CODES team on November 3rd to support theta configuration format -- MM
import sys
import re
import numpy as np
import struct
filename = sys.argv[1]
intracon = open(sys.argv[2], "wb")
intercon = open(sys.argv[3], "wb")
def router(group, row, col):
return group*96 + row*16 + col
numblack = np.zeros((864,864), dtype=np.int)
numblue = np.zeros((864,864), dtype=np.int)
with open(filename) as ofile:
matches = re.findall('c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\).(\w+).->.c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\)', ofile.read(), re.MULTILINE)
print matches
for match in matches:
srcgrp = int(match[0])
#if(srcgrp > 12):
#srcgrp = srcgrp - 1
srcrow = int(match[1])
srccol = int(match[2])
srcrouter = router(srcgrp, srcrow, srccol)
print srcrouter
color = match[3]
dstgrp = int(match[4])
#if(dstgrp > 12):
#dstgrp = dstgrp - 1
dstrow = int(match[5])
dstcol = int(match[6])
dstrouter = router(dstgrp, dstrow, dstcol)
# count number of black and blue links per router pair
if color == 'black':
numblack[srcrouter][dstrouter] += 1
if color == 'blue':
numblue[srcrouter][dstrouter] += 1
if srcgrp == 0:
if color == 'blue':
# write to inter-con file
intercon.write(struct.pack('2i', srcrouter, dstrouter))
# print 'BLUE', srcrouter, dstrouter
else:
# write to intra-con file
if color == 'green':
intracon.write(struct.pack('3i', srcrouter, dstrouter, 0))
# print 'GREEN', srcrouter, dstrouter, 0
elif numblack[srcrouter][dstrouter] < 4:
intracon.write(struct.pack('3i', srcrouter, dstrouter, 1))
# print 'BLACK', srcrouter, dstrouter, 1
else:
if color == 'blue':
# only write the inter-con file
intercon.write(struct.pack('2i', srcrouter, dstrouter))
# print 'BLUE', srcrouter, dstrouter
#for i in range(0, 864):
# for j in range(0, 864):
# if(numblack[i][j] != 0):
# print numblack[i][j],
#print "\n"
#for i in range(0, 864):
# for j in range(0, 864):
# if(numblue[i][j] != 0):
# print numblue[i][j],
intracon.close()
intercon.close()
LPGROUPS
{
MODELNET_GRP
{
repetitions="1600";
# name of this lp changes according to the model
server="4";
# these lp names will be the same for dragonfly-custom model
modelnet_dragonfly_custom="4";
modelnet_dragonfly_custom_router="1";
}
}
PARAMS
{
# packet size in the network
packet_size="1024";
modelnet_order=( "dragonfly_custom","dragonfly_custom_router" );
# scheduler options
modelnet_scheduler="fcfs";
# chunk size in the network (when chunk size = packet size, packets will not be
# divided into chunks)
chunk_size="1024";
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers="80";
# number of groups in the network
num_groups="20";
# buffer size in bytes for local virtual channels
local_vc_size="8192";
#buffer size in bytes for global virtual channels
global_vc_size="16384";
#buffer size in bytes for compute node virtual channels
cn_vc_size="8192";
#bandwidth in GiB/s for local channels
local_bandwidth="5.25";
# bandwidth in GiB/s for global channels
global_bandwidth="18.75";
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth="8.0";
# ROSS message size
message_size="584";
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router="4";
# number of global channels per router
num_global_channels="8";
# network config file for intra-group connections
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom";
# network config file for inter-group connections
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom";
# routing protocol to be used
routing="adaptive";
}
LPGROUPS
{
MODELNET_GRP
{
repetitions="1440";
# name of this lp changes according to the model
server="4";
# these lp names will be the same for dragonfly-custom model
modelnet_dragonfly_custom="4";
modelnet_dragonfly_custom_router="1";
}
}
PARAMS
{
# packet size in the network
packet_size="1024";
modelnet_order=( "dragonfly_custom","dragonfly_custom_router" );
# scheduler options
modelnet_scheduler="fcfs";
# chunk size in the network (when chunk size = packet size, packets will not be
# divided into chunks)
chunk_size="1024";
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers="96";
# number of groups in the network
num_groups="15";
# buffer size in bytes for local virtual channels
local_vc_size="8192";
#buffer size in bytes for global virtual channels
global_vc_size="16384";
#buffer size in bytes for compute node virtual channels
cn_vc_size="8192";
#bandwidth in GiB/s for local channels
local_bandwidth="5.25";
# bandwidth in GiB/s for global channels
global_bandwidth="18.75";
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth="8.0";
# ROSS message size
message_size="584";
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router="4";
# number of global channels per router
num_global_channels="8";
# network config file for intra-group connections
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison";
# network config file for inter-group connections
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison";
# routing protocol to be used
routing="adaptive";
}
......@@ -2,7 +2,7 @@ LPGROUPS
{
MODELNET_GRP
{
repetitions="3200";
repetitions="1600";
# name of this lp changes according to the model
nw-lp="4";
# these lp names will be the same for dragonfly-custom model
......@@ -23,9 +23,9 @@ PARAMS
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers="128";
num_routers="80";
# number of groups in the network
num_groups="25";
num_groups="20";
# buffer size in bytes for local virtual channels
local_vc_size="8192";
#buffer size in bytes for global virtual channels
......@@ -39,16 +39,16 @@ PARAMS
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth="8.0";
# ROSS message size
message_size="584";
message_size="592";
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router="4";
# number of global channels per router
num_global_channels="8";
num_global_channels="10";
# network config file for intra-group connections
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom";
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/intra-custom";
# network config file for inter-group connections
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom";
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/inter-custom";
# routing protocol to be used
routing="prog-adaptive";
}
......@@ -39,16 +39,16 @@ PARAMS
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth="8.0";
# ROSS message size
message_size="584";
message_size="592";
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router="4";
# number of global channels per router
num_global_channels="8";
# network config file for intra-group connections
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison";
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/intra-edison";
# network config file for inter-group connections
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison";
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes-conflict/scripts/gen-cray-topo/inter-edison";
# routing protocol to be used
routing="non-minimal";
}
......@@ -35,20 +35,20 @@ PARAMS
#bandwidth in GiB/s for local channels
local_bandwidth="5.25";
# bandwidth in GiB/s for global channels
global_bandwidth="4.7";
global_bandwidth="18.75";
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth="16.0";
# ROSS message size
message_size="584";
message_size="592";
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router="4";
# number of global channels per router
num_global_channels="12";
num_global_channels="10";
# network config file for intra-group connections
intra-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-theta";
intra-group-connections="/Users/mmubarak/Documents/software_development/test_versions/codes/scripts/gen-cray-topo/intra-theta";
# network config file for inter-group connections
inter-group-connections="/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-theta";
inter-group-connections="/Users/mmubarak/Documents/software_development/test_versions/codes/scripts/gen-cray-topo/inter-theta";
# routing protocol to be used
routing="prog-adaptive";
}
......@@ -1640,7 +1640,6 @@ int main( int argc, char** argv )
num_mpi_lps = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 0);
num_net_lps = codes_mapping_get_lp_count("MODELNET_GRP", 1, MN_LP_NM, NULL, 0);
printf("\n num net lps %d ", num_net_lps);
if (lp_io_dir[0]){
do_lp_io = 1;
/* initialize lp io */
......
......@@ -126,7 +126,7 @@ const tw_lptype* svr_get_lp_type()
static void svr_add_lp_type()
{
lp_type_register("server", svr_get_lp_type());
lp_type_register("nw-lp", svr_get_lp_type());
}
static void issue_event(
......@@ -382,7 +382,7 @@ int main(
MPI_Finalize();
return 0;
}
num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "server",
num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "nw-lp",
NULL, 1);
configuration_get_value_int(&config, "PARAMS", "num_routers", NULL, &num_routers_per_grp);
configuration_get_value_int(&config, "PARAMS", "num_groups", NULL, &num_groups);
......
......@@ -60,12 +60,19 @@
20 groups. Each group has 80 routers arranged in a 20x4 matrix]
./bin/model-net-synthetic-custom-dfly --sync=1 --
../src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-custom.conf
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf
[With edison dragonfly network have 5,702 network nodes, 1440 routers and 15
[With theta dragonfly network having 3,456 compute nodes, 864 routers and 9
groups. Each group has 96 routers arranged in a 6x16 matrix]
mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf
[With edison dragonfly network having 5,702 network nodes, 1440 routers and 15
groups. Each group has 96 routers arranged in 6x16 matrix.]
mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
../src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-edison.conf
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-edison.conf
- Design Forward Network traces:
......@@ -79,12 +86,19 @@
[With Edison style dragonfly and AMG 1,728 application trace]
./bin/model-net-mpi-replay --sync=1 --disable_compute=1
mpirun -np 4 ./bin/model-net-mpi-replay --sync=3 --disable_compute=1
--workload_type="dumpi"
--workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
--num_net_traces=1728 --
../src/network-workloads/conf/modelnet-test-dragonfly-edison.conf
../src/network-workloads/dragonfly-custom/conf/modelnet-test-dragonfly-edison.conf
[With theta style dragonfly and AMG 1,728 application trace]
mpirun -np 4 ./bin/model-net-mpi-replay --sync=3 --disable_compute=1
--workload_type="dumpi"
--workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
--num_net_traces=1728 --
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-
--------- Debugging Tips ------------
- Set DUMP_CONNECTIONS debugging option to see the detailed local and global
channel connectivity of routers in src/networks/model-net/dragonfly-custom.C
......@@ -20,7 +20,7 @@
#include <vector>
#include <map>
#define DUMP_CONNECTIONS 0
#define DUMP_CONNECTIONS 1
#define CREDIT_SIZE 8
#define DFLY_HASH_TABLE_SIZE 262144
......@@ -282,6 +282,11 @@ enum ROUTING_ALGO
PROG_ADAPTIVE
};
enum LINK_TYPE
{
GREEN,
BLACK,
};
struct router_state
{
unsigned int router_id;
......@@ -690,9 +695,9 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
}
#endif
if(!myRank) {
printf("\n Total nodes %d routers %d groups %d radix %d \n",
printf("\n Total nodes %d routers %d groups %d routers per group %d\n",
p->num_cn * p->total_routers, p->total_routers, p->num_groups,
p->radix);
p->num_routers);
}
p->cn_delay = bytes_to_ns(p->chunk_size, p->cn_bandwidth);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment