Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Caitlin Ross
codes
Commits
29bd0c1e
Commit
29bd0c1e
authored
Nov 03, 2016
by
Misbah Mubarak
Browse files
Adding theta network configuration
parent
abcccfe0
Changes
12
Expand all
Hide whitespace changes
Inline
Side-by-side
scripts/gen-cray-topo/README.txt
View file @
29bd0c1e
...
...
@@ -8,6 +8,13 @@
translates Edison's network configuration into a file format that can be fed
into the simulation.]
- Theta network config files:
python theta.py theta.interconnect intra-theta inter-theta
[intra-theta and inter-theta are the intra and inter-group config files for
dragonfly. ]
** Generating customizable dragonfly interconnects **:
mpicc connections_general.c -o connections_general
...
...
scripts/gen-cray-topo/theta.interconnect
0 → 100644
View file @
29bd0c1e
This diff is collapsed.
Click to expand it.
scripts/gen-cray-topo/theta.py
0 → 100644
View file @
29bd0c1e
#!/usr/bin/env python
##############################################################################
# Copyright (c) 2014, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# Written by:
# Nikhil Jain <nikhil.jain@acm.org>
# Abhinav Bhatele <bhatele@llnl.gov>
# Peer-Timo Bremer <ptbremer@llnl.gov>
#
# LLNL-CODE-678961. All rights reserved.
#
# This file is part of Damselfly. For details, see:
# https://github.com/LLNL/damselfly
# Please also read the LICENSE file for our notice and the LGPL.
##############################################################################
# Modified by CODES team on November 3rd to support theta configuration format -- MM
import
sys
import
re
import
numpy
as
np
import
struct
filename
=
sys
.
argv
[
1
]
intracon
=
open
(
sys
.
argv
[
2
],
"wb"
)
intercon
=
open
(
sys
.
argv
[
3
],
"wb"
)
def
router
(
group
,
row
,
col
):
return
group
*
96
+
row
*
16
+
col
numblack
=
np
.
zeros
((
864
,
864
),
dtype
=
np
.
int
)
numblue
=
np
.
zeros
((
864
,
864
),
dtype
=
np
.
int
)
with
open
(
filename
)
as
ofile
:
matches
=
re
.
findall
(
'c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\).(\w+).->.c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\)'
,
ofile
.
read
(),
re
.
MULTILINE
)
print
matches
for
match
in
matches
:
srcgrp
=
int
(
match
[
0
])
#if(srcgrp > 12):
#srcgrp = srcgrp - 1
srcrow
=
int
(
match
[
1
])
srccol
=
int
(
match
[
2
])
srcrouter
=
router
(
srcgrp
,
srcrow
,
srccol
)
print
srcrouter
color
=
match
[
3
]
dstgrp
=
int
(
match
[
4
])
#if(dstgrp > 12):
#dstgrp = dstgrp - 1
dstrow
=
int
(
match
[
5
])
dstcol
=
int
(
match
[
6
])
dstrouter
=
router
(
dstgrp
,
dstrow
,
dstcol
)
# count number of black and blue links per router pair
if
color
==
'black'
:
numblack
[
srcrouter
][
dstrouter
]
+=
1
if
color
==
'blue'
:
numblue
[
srcrouter
][
dstrouter
]
+=
1
if
srcgrp
==
0
:
if
color
==
'blue'
:
# write to inter-con file
intercon
.
write
(
struct
.
pack
(
'2i'
,
srcrouter
,
dstrouter
))
# print 'BLUE', srcrouter, dstrouter
else
:
# write to intra-con file
if
color
==
'green'
:
intracon
.
write
(
struct
.
pack
(
'3i'
,
srcrouter
,
dstrouter
,
0
))
# print 'GREEN', srcrouter, dstrouter, 0
elif
numblack
[
srcrouter
][
dstrouter
]
<
4
:
intracon
.
write
(
struct
.
pack
(
'3i'
,
srcrouter
,
dstrouter
,
1
))
# print 'BLACK', srcrouter, dstrouter, 1
else
:
if
color
==
'blue'
:
# only write the inter-con file
intercon
.
write
(
struct
.
pack
(
'2i'
,
srcrouter
,
dstrouter
))
# print 'BLUE', srcrouter, dstrouter
#for i in range(0, 864):
# for j in range(0, 864):
# if(numblack[i][j] != 0):
# print numblack[i][j],
#print "\n"
#for i in range(0, 864):
# for j in range(0, 864):
# if(numblue[i][j] != 0):
# print numblue[i][j],
intracon
.
close
()
intercon
.
close
()
src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-custom.conf
deleted
100644 → 0
View file @
abcccfe0
LPGROUPS
{
MODELNET_GRP
{
repetitions
=
"1600"
;
# name of this lp changes according to the model
server
=
"4"
;
# these lp names will be the same for dragonfly-custom model
modelnet_dragonfly_custom
=
"4"
;
modelnet_dragonfly_custom_router
=
"1"
;
}
}
PARAMS
{
# packet size in the network
packet_size
=
"1024"
;
modelnet_order
=(
"dragonfly_custom"
,
"dragonfly_custom_router"
);
# scheduler options
modelnet_scheduler
=
"fcfs"
;
# chunk size in the network (when chunk size = packet size, packets will not be
# divided into chunks)
chunk_size
=
"1024"
;
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers
=
"80"
;
# number of groups in the network
num_groups
=
"20"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
global_vc_size
=
"16384"
;
#buffer size in bytes for compute node virtual channels
cn_vc_size
=
"8192"
;
#bandwidth in GiB/s for local channels
local_bandwidth
=
"5.25"
;
# bandwidth in GiB/s for global channels
global_bandwidth
=
"18.75"
;
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"584"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"8"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom"
;
# routing protocol to be used
routing
=
"adaptive"
;
}
src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-edison.conf
deleted
100644 → 0
View file @
abcccfe0
LPGROUPS
{
MODELNET_GRP
{
repetitions
=
"1440"
;
# name of this lp changes according to the model
server
=
"4"
;
# these lp names will be the same for dragonfly-custom model
modelnet_dragonfly_custom
=
"4"
;
modelnet_dragonfly_custom_router
=
"1"
;
}
}
PARAMS
{
# packet size in the network
packet_size
=
"1024"
;
modelnet_order
=(
"dragonfly_custom"
,
"dragonfly_custom_router"
);
# scheduler options
modelnet_scheduler
=
"fcfs"
;
# chunk size in the network (when chunk size = packet size, packets will not be
# divided into chunks)
chunk_size
=
"1024"
;
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers
=
"96"
;
# number of groups in the network
num_groups
=
"15"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
global_vc_size
=
"16384"
;
#buffer size in bytes for compute node virtual channels
cn_vc_size
=
"8192"
;
#bandwidth in GiB/s for local channels
local_bandwidth
=
"5.25"
;
# bandwidth in GiB/s for global channels
global_bandwidth
=
"18.75"
;
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"584"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"8"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison"
;
# routing protocol to be used
routing
=
"adaptive"
;
}
src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf
View file @
29bd0c1e
...
...
@@ -2,7 +2,7 @@ LPGROUPS
{
MODELNET_GRP
{
repetitions
=
"
32
00"
;
repetitions
=
"
16
00"
;
# name of this lp changes according to the model
nw
-
lp
=
"4"
;
# these lp names will be the same for dragonfly-custom model
...
...
@@ -23,9 +23,9 @@ PARAMS
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers
=
"
12
8"
;
num_routers
=
"8
0
"
;
# number of groups in the network
num_groups
=
"2
5
"
;
num_groups
=
"2
0
"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
...
...
@@ -39,16 +39,16 @@ PARAMS
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"5
84
"
;
message_size
=
"5
92
"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"
8
"
;
num_global_channels
=
"
10
"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom"
;
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/intra-custom"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom"
;
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/inter-custom"
;
# routing protocol to be used
routing
=
"prog-adaptive"
;
}
src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-edison.conf
View file @
29bd0c1e
...
...
@@ -39,16 +39,16 @@ PARAMS
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"5
84
"
;
message_size
=
"5
92
"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"8"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison"
;
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/intra-edison"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison"
;
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/inter-edison"
;
# routing protocol to be used
routing
=
"non-minimal"
;
}
src/network-workloads/conf/modelnet-test-dragonfly-theta.conf
→
src/network-workloads/conf/
dragonfly-custom/
modelnet-test-dragonfly-theta.conf
View file @
29bd0c1e
...
...
@@ -35,20 +35,20 @@ PARAMS
#bandwidth in GiB/s for local channels
local_bandwidth
=
"5.25"
;
# bandwidth in GiB/s for global channels
global_bandwidth
=
"
4
.7"
;
global_bandwidth
=
"
18
.7
5
"
;
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"16.0"
;
# ROSS message size
message_size
=
"5
84
"
;
message_size
=
"5
92
"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"1
2
"
;
num_global_channels
=
"1
0
"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
dragonfly-cray
/codes/scripts/gen-cray-topo/intra-theta"
;
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
test_versions
/codes/scripts/gen-cray-topo/intra-theta"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
dragonfly-cray
/codes/scripts/gen-cray-topo/inter-theta"
;
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
test_versions
/codes/scripts/gen-cray-topo/inter-theta"
;
# routing protocol to be used
routing
=
"prog-adaptive"
;
}
src/network-workloads/model-net-mpi-replay.c
View file @
29bd0c1e
...
...
@@ -1640,7 +1640,6 @@ int main( int argc, char** argv )
num_mpi_lps
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
0
,
"nw-lp"
,
NULL
,
0
);
num_net_lps
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
MN_LP_NM
,
NULL
,
0
);
printf
(
"
\n
num net lps %d "
,
num_net_lps
);
if
(
lp_io_dir
[
0
]){
do_lp_io
=
1
;
/* initialize lp io */
...
...
src/network-workloads/model-net-synthetic-custom-dfly.c
View file @
29bd0c1e
...
...
@@ -126,7 +126,7 @@ const tw_lptype* svr_get_lp_type()
static
void
svr_add_lp_type
()
{
lp_type_register
(
"
server
"
,
svr_get_lp_type
());
lp_type_register
(
"
nw-lp
"
,
svr_get_lp_type
());
}
static
void
issue_event
(
...
...
@@ -382,7 +382,7 @@ int main(
MPI_Finalize
();
return
0
;
}
num_servers_per_rep
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
"
server
"
,
num_servers_per_rep
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
"
nw-lp
"
,
NULL
,
1
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_routers"
,
NULL
,
&
num_routers_per_grp
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_groups"
,
NULL
,
&
num_groups
);
...
...
src/networks/model-net/doc/README.dragonfly-custom.txt
View file @
29bd0c1e
...
...
@@ -60,12 +60,19 @@
20 groups. Each group has 80 routers arranged in a 20x4 matrix]
./bin/model-net-synthetic-custom-dfly --sync=1 --
../src/network-workloads/conf/dragonfly-custom/modelnet-
synthetic
-custom.conf
../src/network-workloads/conf/dragonfly-custom/modelnet-
test-dragonfly
-custom.conf
[With edison dragonfly network have 5,702 network nodes, 1440 routers and 15
[With theta dragonfly network having 3,456 compute nodes, 864 routers and 9
groups. Each group has 96 routers arranged in a 6x16 matrix]
mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf
[With edison dragonfly network having 5,702 network nodes, 1440 routers and 15
groups. Each group has 96 routers arranged in 6x16 matrix.]
mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
../src/network-workloads/conf/dragonfly-custom/modelnet-
synthetic
-edison.conf
../src/network-workloads/conf/dragonfly-custom/modelnet-
test-dragonfly
-edison.conf
- Design Forward Network traces:
...
...
@@ -79,12 +86,19 @@
[With Edison style dragonfly and AMG 1,728 application trace]
./bin/model-net-mpi-replay --sync=
1
--disable_compute=1
mpirun -np 4
./bin/model-net-mpi-replay --sync=
3
--disable_compute=1
--workload_type="dumpi"
--workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
--num_net_traces=1728 --
../src/network-workloads/conf/modelnet-test-dragonfly-edison.conf
../src/network-workloads/
dragonfly-custom/
conf/modelnet-test-dragonfly-edison.conf
[With theta style dragonfly and AMG 1,728 application trace]
mpirun -np 4 ./bin/model-net-mpi-replay --sync=3 --disable_compute=1
--workload_type="dumpi"
--workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
--num_net_traces=1728 --
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-
--------- Debugging Tips ------------
- Set DUMP_CONNECTIONS debugging option to see the detailed local and global
channel connectivity of routers in src/networks/model-net/dragonfly-custom.C
src/networks/model-net/dragonfly-custom.C
View file @
29bd0c1e
...
...
@@ -20,7 +20,7 @@
#include
<vector>
#include
<map>
#define DUMP_CONNECTIONS
0
#define DUMP_CONNECTIONS
1
#define CREDIT_SIZE 8
#define DFLY_HASH_TABLE_SIZE 262144
...
...
@@ -282,6 +282,11 @@ enum ROUTING_ALGO
PROG_ADAPTIVE
};
enum
LINK_TYPE
{
GREEN
,
BLACK
,
};
struct
router_state
{
unsigned
int
router_id
;
...
...
@@ -690,9 +695,9 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
}
#endif
if
(
!
myRank
)
{
printf
(
"
\n
Total nodes %d routers %d groups %d r
adix
%d
\n
"
,
printf
(
"
\n
Total nodes %d routers %d groups %d r
outers per group
%d
\n
"
,
p
->
num_cn
*
p
->
total_routers
,
p
->
total_routers
,
p
->
num_groups
,
p
->
radix
);
p
->
num_routers
);
}
p
->
cn_delay
=
bytes_to_ns
(
p
->
chunk_size
,
p
->
cn_bandwidth
);
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment