Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Xin Wang
codes-dev
Commits
29bd0c1e
Commit
29bd0c1e
authored
Nov 03, 2016
by
Misbah Mubarak
Browse files
Adding theta network configuration
parent
abcccfe0
Changes
12
Expand all
Hide whitespace changes
Inline
Side-by-side
scripts/gen-cray-topo/README.txt
View file @
29bd0c1e
...
...
@@ -8,6 +8,13 @@
translates Edison's network configuration into a file format that can be fed
into the simulation.]
- Theta network config files:
python theta.py theta.interconnect intra-theta inter-theta
[intra-theta and inter-theta are the intra and inter-group config files for
dragonfly. ]
** Generating customizable dragonfly interconnects **:
mpicc connections_general.c -o connections_general
...
...
scripts/gen-cray-topo/theta.interconnect
0 → 100644
View file @
29bd0c1e
This diff is collapsed.
Click to expand it.
scripts/gen-cray-topo/theta.py
0 → 100644
View file @
29bd0c1e
#!/usr/bin/env python
##############################################################################
# Copyright (c) 2014, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# Written by:
# Nikhil Jain <nikhil.jain@acm.org>
# Abhinav Bhatele <bhatele@llnl.gov>
# Peer-Timo Bremer <ptbremer@llnl.gov>
#
# LLNL-CODE-678961. All rights reserved.
#
# This file is part of Damselfly. For details, see:
# https://github.com/LLNL/damselfly
# Please also read the LICENSE file for our notice and the LGPL.
##############################################################################
# Modified by CODES team on November 3rd to support theta configuration format -- MM
import
sys
import
re
import
numpy
as
np
import
struct
filename
=
sys
.
argv
[
1
]
intracon
=
open
(
sys
.
argv
[
2
],
"wb"
)
intercon
=
open
(
sys
.
argv
[
3
],
"wb"
)
def
router
(
group
,
row
,
col
):
return
group
*
96
+
row
*
16
+
col
numblack
=
np
.
zeros
((
864
,
864
),
dtype
=
np
.
int
)
numblue
=
np
.
zeros
((
864
,
864
),
dtype
=
np
.
int
)
with
open
(
filename
)
as
ofile
:
matches
=
re
.
findall
(
'c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\).(\w+).->.c\d+-\dc\ds\d+a0l\d+\((\d+):(\d):(\d+)\)'
,
ofile
.
read
(),
re
.
MULTILINE
)
print
matches
for
match
in
matches
:
srcgrp
=
int
(
match
[
0
])
#if(srcgrp > 12):
#srcgrp = srcgrp - 1
srcrow
=
int
(
match
[
1
])
srccol
=
int
(
match
[
2
])
srcrouter
=
router
(
srcgrp
,
srcrow
,
srccol
)
print
srcrouter
color
=
match
[
3
]
dstgrp
=
int
(
match
[
4
])
#if(dstgrp > 12):
#dstgrp = dstgrp - 1
dstrow
=
int
(
match
[
5
])
dstcol
=
int
(
match
[
6
])
dstrouter
=
router
(
dstgrp
,
dstrow
,
dstcol
)
# count number of black and blue links per router pair
if
color
==
'black'
:
numblack
[
srcrouter
][
dstrouter
]
+=
1
if
color
==
'blue'
:
numblue
[
srcrouter
][
dstrouter
]
+=
1
if
srcgrp
==
0
:
if
color
==
'blue'
:
# write to inter-con file
intercon
.
write
(
struct
.
pack
(
'2i'
,
srcrouter
,
dstrouter
))
# print 'BLUE', srcrouter, dstrouter
else
:
# write to intra-con file
if
color
==
'green'
:
intracon
.
write
(
struct
.
pack
(
'3i'
,
srcrouter
,
dstrouter
,
0
))
# print 'GREEN', srcrouter, dstrouter, 0
elif
numblack
[
srcrouter
][
dstrouter
]
<
4
:
intracon
.
write
(
struct
.
pack
(
'3i'
,
srcrouter
,
dstrouter
,
1
))
# print 'BLACK', srcrouter, dstrouter, 1
else
:
if
color
==
'blue'
:
# only write the inter-con file
intercon
.
write
(
struct
.
pack
(
'2i'
,
srcrouter
,
dstrouter
))
# print 'BLUE', srcrouter, dstrouter
#for i in range(0, 864):
# for j in range(0, 864):
# if(numblack[i][j] != 0):
# print numblack[i][j],
#print "\n"
#for i in range(0, 864):
# for j in range(0, 864):
# if(numblue[i][j] != 0):
# print numblue[i][j],
intracon
.
close
()
intercon
.
close
()
src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-custom.conf
deleted
100644 → 0
View file @
abcccfe0
LPGROUPS
{
MODELNET_GRP
{
repetitions
=
"1600"
;
# name of this lp changes according to the model
server
=
"4"
;
# these lp names will be the same for dragonfly-custom model
modelnet_dragonfly_custom
=
"4"
;
modelnet_dragonfly_custom_router
=
"1"
;
}
}
PARAMS
{
# packet size in the network
packet_size
=
"1024"
;
modelnet_order
=(
"dragonfly_custom"
,
"dragonfly_custom_router"
);
# scheduler options
modelnet_scheduler
=
"fcfs"
;
# chunk size in the network (when chunk size = packet size, packets will not be
# divided into chunks)
chunk_size
=
"1024"
;
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers
=
"80"
;
# number of groups in the network
num_groups
=
"20"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
global_vc_size
=
"16384"
;
#buffer size in bytes for compute node virtual channels
cn_vc_size
=
"8192"
;
#bandwidth in GiB/s for local channels
local_bandwidth
=
"5.25"
;
# bandwidth in GiB/s for global channels
global_bandwidth
=
"18.75"
;
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"584"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"8"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom"
;
# routing protocol to be used
routing
=
"adaptive"
;
}
src/network-workloads/conf/dragonfly-custom/modelnet-synthetic-edison.conf
deleted
100644 → 0
View file @
abcccfe0
LPGROUPS
{
MODELNET_GRP
{
repetitions
=
"1440"
;
# name of this lp changes according to the model
server
=
"4"
;
# these lp names will be the same for dragonfly-custom model
modelnet_dragonfly_custom
=
"4"
;
modelnet_dragonfly_custom_router
=
"1"
;
}
}
PARAMS
{
# packet size in the network
packet_size
=
"1024"
;
modelnet_order
=(
"dragonfly_custom"
,
"dragonfly_custom_router"
);
# scheduler options
modelnet_scheduler
=
"fcfs"
;
# chunk size in the network (when chunk size = packet size, packets will not be
# divided into chunks)
chunk_size
=
"1024"
;
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers
=
"96"
;
# number of groups in the network
num_groups
=
"15"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
global_vc_size
=
"16384"
;
#buffer size in bytes for compute node virtual channels
cn_vc_size
=
"8192"
;
#bandwidth in GiB/s for local channels
local_bandwidth
=
"5.25"
;
# bandwidth in GiB/s for global channels
global_bandwidth
=
"18.75"
;
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"584"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"8"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison"
;
# routing protocol to be used
routing
=
"adaptive"
;
}
src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-custom.conf
View file @
29bd0c1e
...
...
@@ -2,7 +2,7 @@ LPGROUPS
{
MODELNET_GRP
{
repetitions
=
"
32
00"
;
repetitions
=
"
16
00"
;
# name of this lp changes according to the model
nw
-
lp
=
"4"
;
# these lp names will be the same for dragonfly-custom model
...
...
@@ -23,9 +23,9 @@ PARAMS
# modelnet_scheduler="round-robin";
# number of routers within each group
# this is dictated by the dragonfly configuration files
num_routers
=
"
12
8"
;
num_routers
=
"8
0
"
;
# number of groups in the network
num_groups
=
"2
5
"
;
num_groups
=
"2
0
"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
...
...
@@ -39,16 +39,16 @@ PARAMS
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"5
84
"
;
message_size
=
"5
92
"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"
8
"
;
num_global_channels
=
"
10
"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-custom"
;
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/intra-custom"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-custom"
;
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/inter-custom"
;
# routing protocol to be used
routing
=
"prog-adaptive"
;
}
src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-edison.conf
View file @
29bd0c1e
...
...
@@ -39,16 +39,16 @@ PARAMS
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"8.0"
;
# ROSS message size
message_size
=
"5
84
"
;
message_size
=
"5
92
"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"8"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/intra-edison"
;
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/intra-edison"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes/scripts/gen-cray-topo/inter-edison"
;
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/dragonfly-cray/codes
-conflict
/scripts/gen-cray-topo/inter-edison"
;
# routing protocol to be used
routing
=
"non-minimal"
;
}
src/network-workloads/conf/modelnet-test-dragonfly-theta.conf
→
src/network-workloads/conf/
dragonfly-custom/
modelnet-test-dragonfly-theta.conf
View file @
29bd0c1e
...
...
@@ -35,20 +35,20 @@ PARAMS
#bandwidth in GiB/s for local channels
local_bandwidth
=
"5.25"
;
# bandwidth in GiB/s for global channels
global_bandwidth
=
"
4
.7"
;
global_bandwidth
=
"
18
.7
5
"
;
# bandwidth in GiB/s for compute node-router channels
cn_bandwidth
=
"16.0"
;
# ROSS message size
message_size
=
"5
84
"
;
message_size
=
"5
92
"
;
# number of compute nodes connected to router, dictated by dragonfly config
# file
num_cns_per_router
=
"4"
;
# number of global channels per router
num_global_channels
=
"1
2
"
;
num_global_channels
=
"1
0
"
;
# network config file for intra-group connections
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
dragonfly-cray
/codes/scripts/gen-cray-topo/intra-theta"
;
intra
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
test_versions
/codes/scripts/gen-cray-topo/intra-theta"
;
# network config file for inter-group connections
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
dragonfly-cray
/codes/scripts/gen-cray-topo/inter-theta"
;
inter
-
group
-
connections
=
"/Users/mmubarak/Documents/software_development/
test_versions
/codes/scripts/gen-cray-topo/inter-theta"
;
# routing protocol to be used
routing
=
"prog-adaptive"
;
}
src/network-workloads/model-net-mpi-replay.c
View file @
29bd0c1e
...
...
@@ -1640,7 +1640,6 @@ int main( int argc, char** argv )
num_mpi_lps
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
0
,
"nw-lp"
,
NULL
,
0
);
num_net_lps
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
MN_LP_NM
,
NULL
,
0
);
printf
(
"
\n
num net lps %d "
,
num_net_lps
);
if
(
lp_io_dir
[
0
]){
do_lp_io
=
1
;
/* initialize lp io */
...
...
src/network-workloads/model-net-synthetic-custom-dfly.c
View file @
29bd0c1e
...
...
@@ -126,7 +126,7 @@ const tw_lptype* svr_get_lp_type()
static
void
svr_add_lp_type
()
{
lp_type_register
(
"
server
"
,
svr_get_lp_type
());
lp_type_register
(
"
nw-lp
"
,
svr_get_lp_type
());
}
static
void
issue_event
(
...
...
@@ -382,7 +382,7 @@ int main(
MPI_Finalize
();
return
0
;
}
num_servers_per_rep
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
"
server
"
,
num_servers_per_rep
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
"
nw-lp
"
,
NULL
,
1
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_routers"
,
NULL
,
&
num_routers_per_grp
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_groups"
,
NULL
,
&
num_groups
);
...
...
src/networks/model-net/doc/README.dragonfly-custom.txt
View file @
29bd0c1e
...
...
@@ -60,12 +60,19 @@
20 groups. Each group has 80 routers arranged in a 20x4 matrix]
./bin/model-net-synthetic-custom-dfly --sync=1 --
../src/network-workloads/conf/dragonfly-custom/modelnet-
synthetic
-custom.conf
../src/network-workloads/conf/dragonfly-custom/modelnet-
test-dragonfly
-custom.conf
[With edison dragonfly network have 5,702 network nodes, 1440 routers and 15
[With theta dragonfly network having 3,456 compute nodes, 864 routers and 9
groups. Each group has 96 routers arranged in a 6x16 matrix]
mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-theta.conf
[With edison dragonfly network having 5,702 network nodes, 1440 routers and 15
groups. Each group has 96 routers arranged in 6x16 matrix.]
mpirun -np 4 ./bin/model-net-synthetic-custom-dfly --sync=3 --
../src/network-workloads/conf/dragonfly-custom/modelnet-
synthetic
-edison.conf
../src/network-workloads/conf/dragonfly-custom/modelnet-
test-dragonfly
-edison.conf
- Design Forward Network traces:
...
...
@@ -79,12 +86,19 @@
[With Edison style dragonfly and AMG 1,728 application trace]
./bin/model-net-mpi-replay --sync=
1
--disable_compute=1
mpirun -np 4
./bin/model-net-mpi-replay --sync=
3
--disable_compute=1
--workload_type="dumpi"
--workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
--num_net_traces=1728 --
../src/network-workloads/conf/modelnet-test-dragonfly-edison.conf
../src/network-workloads/
dragonfly-custom/
conf/modelnet-test-dragonfly-edison.conf
[With theta style dragonfly and AMG 1,728 application trace]
mpirun -np 4 ./bin/model-net-mpi-replay --sync=3 --disable_compute=1
--workload_type="dumpi"
--workload_file=../../../df_traces/AMG/df_AMG_n1728_dumpi/dumpi-2014.03.03.14.55.50-
--num_net_traces=1728 --
../src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-
--------- Debugging Tips ------------
- Set DUMP_CONNECTIONS debugging option to see the detailed local and global
channel connectivity of routers in src/networks/model-net/dragonfly-custom.C
src/networks/model-net/dragonfly-custom.C
View file @
29bd0c1e
...
...
@@ -20,7 +20,7 @@
#include <vector>
#include <map>
#define DUMP_CONNECTIONS
0
#define DUMP_CONNECTIONS
1
#define CREDIT_SIZE 8
#define DFLY_HASH_TABLE_SIZE 262144
...
...
@@ -282,6 +282,11 @@ enum ROUTING_ALGO
PROG_ADAPTIVE
};
enum
LINK_TYPE
{
GREEN
,
BLACK
,
};
struct
router_state
{
unsigned
int
router_id
;
...
...
@@ -690,9 +695,9 @@ static void dragonfly_read_config(const char * anno, dragonfly_param *params){
}
#endif
if
(
!
myRank
)
{
printf
(
"
\n
Total nodes %d routers %d groups %d r
adix
%d
\n
"
,
printf
(
"
\n
Total nodes %d routers %d groups %d r
outers per group
%d
\n
"
,
p
->
num_cn
*
p
->
total_routers
,
p
->
total_routers
,
p
->
num_groups
,
p
->
radix
);
p
->
num_routers
);
}
p
->
cn_delay
=
bytes_to_ns
(
p
->
chunk_size
,
p
->
cn_bandwidth
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment