Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
C
codes
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
38
Issues
38
List
Boards
Labels
Milestones
Merge Requests
8
Merge Requests
8
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
codes
codes
Commits
78d73dda
Commit
78d73dda
authored
Jan 30, 2018
by
Neil McGlohon
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Dragonfly Plus: Model, Workload, and Config change
parent
e602f6bc
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
421 additions
and
471 deletions
+421
-471
src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-plus-testing.conf
...ragonfly-custom/modelnet-test-dragonfly-plus-testing.conf
+4
-0
src/network-workloads/model-net-synthetic-dfly-plus.c
src/network-workloads/model-net-synthetic-dfly-plus.c
+8
-6
src/networks/model-net/dragonfly-plus.C
src/networks/model-net/dragonfly-plus.C
+409
-465
No files found.
src/network-workloads/conf/dragonfly-custom/modelnet-test-dragonfly-plus-testing.conf
View file @
78d73dda
...
...
@@ -30,6 +30,10 @@ PARAMS
num_level_chans
=
"1"
;
# number of groups in the network
num_groups
=
"4"
;
# predefined threshold (T) deciding when to reassign packet to a lower priority queue
queue_threshold
=
"50"
;
# buffer size in bytes for local virtual channels
local_vc_size
=
"8192"
;
#buffer size in bytes for global virtual channels
...
...
src/network-workloads/model-net-synthetic-dfly-plus.c
View file @
78d73dda
...
...
@@ -360,7 +360,8 @@ int main(
int
rank
;
int
num_nets
;
int
*
net_ids
;
int
num_router_rows
,
num_router_cols
;
int
num_routers
;
int
num_router_leaf
,
num_router_spine
;
tw_opt_add
(
app_opt
);
tw_init
(
&
argc
,
&
argv
);
...
...
@@ -399,15 +400,16 @@ int main(
}
num_servers_per_rep
=
codes_mapping_get_lp_count
(
"MODELNET_GRP"
,
1
,
"nw-lp"
,
NULL
,
1
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_router_rows"
,
NULL
,
&
num_router_rows
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_router_cols"
,
NULL
,
&
num_router_cols
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_router_leaf"
,
NULL
,
&
num_router_leaf
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_router_spine"
,
NULL
,
&
num_router_spine
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_routers"
,
NULL
,
&
num_routers
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_groups"
,
NULL
,
&
num_groups
);
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_cns_per_router"
,
NULL
,
&
num_nodes_per_cn
);
num_routers_per_grp
=
num_router
_rows
*
num_router_col
s
;
num_routers_per_grp
=
num_routers
;
num_nodes
=
num_groups
*
num_router
s_per_grp
*
num_nodes_per_cn
;
num_nodes_per_grp
=
num_router
s_per_grp
*
num_nodes_per_cn
;
num_nodes
=
num_groups
*
num_router
_leaf
*
num_nodes_per_cn
;
num_nodes_per_grp
=
num_router
_leaf
*
num_nodes_per_cn
;
assert
(
num_nodes
);
...
...
src/networks/model-net/dragonfly-plus.C
View file @
78d73dda
...
...
@@ -54,7 +54,7 @@ struct bLink
/* Each entry in the vector is for a router id
* against each router id, there is a map of links (key of the map is the dest
* router id)
* link has information o
n type (green or black) and o
ffset (number of links
* link has information offset (number of links
* between that particular source and dest router ID)*/
static
vector
<
map
<
int
,
vector
<
Link
>
>
>
intraGroupLinks
;
...
...
@@ -172,13 +172,14 @@ struct dragonfly_plus_param
int
num_level_chans
;
// number of channels between levels of the group(?)
int
num_router_spine
;
// number of spine routers (top level)
int
num_router_leaf
;
// number of leaf routers (bottom level)
int
queue_threshold
;
// predefined queue length threshold T before a packet is routed through a lower priority queue
// dfp params end
int
num_groups
;
int
radix
;
int
total_routers
;
int
total_terminals
;
int
num_global_c
hannel
s
;
int
num_global_c
onnection
s
;
double
cn_delay
;
double
local_delay
;
double
global_delay
;
...
...
@@ -319,9 +320,7 @@ enum last_hop
enum
ROUTING_ALGO
{
MINIMAL
=
1
,
NON_MINIMAL
,
ADAPTIVE
,
PROG_ADAPTIVE
FULLY_PROG_ADAPTIVE
};
enum
LINK_TYPE
...
...
@@ -343,14 +342,6 @@ struct router_state
int
op_arr_size
;
int
max_arr_size
;
/* radix properties - these are here because not all routers will necessarily have the same
* number of ports utilized for inter and intra connections.
* (e.g. Spines have global connections whereas leafs don't) */
int
radix_util
;
//total ports actually utilized
int
cn_radix_util
;
//total ports actually utilized for compute node/terminal connections -- unique to leafs
int
intra_radix_util
;
//total ports actually utilized for intra-group connections -- leafs and spines both have some
int
inter_radix_util
;
//total ports actually utilized for inter-group connections -- unique to spines
router_type
dfp_router_type
;
// Enum to specify whether this router is a spine or a leaf
int
*
global_channel
;
...
...
@@ -589,15 +580,11 @@ static void dragonfly_read_config(const char *anno, dragonfly_plus_param *params
configuration_get_value
(
&
config
,
"PARAMS"
,
"routing"
,
anno
,
routing_str
,
MAX_NAME_LENGTH
);
if
(
strcmp
(
routing_str
,
"minimal"
)
==
0
)
routing
=
MINIMAL
;
else
if
(
strcmp
(
routing_str
,
"nonminimal"
)
==
0
||
strcmp
(
routing_str
,
"non-minimal"
)
==
0
)
routing
=
NON_MINIMAL
;
else
if
(
strcmp
(
routing_str
,
"adaptive"
)
==
0
)
routing
=
ADAPTIVE
;
else
if
(
strcmp
(
routing_str
,
"prog-adaptive"
)
==
0
)
routing
=
PROG_ADAPTIVE
;
else
if
(
strcmp
(
routing_str
,
"fully-prog-adaptive"
)
==
0
)
routing
=
FULLY_PROG_ADAPTIVE
;
else
{
fprintf
(
stderr
,
"No routing protocol specified, setting to minimal routing
\n
"
);
routing
=
-
1
;
routing
=
MINIMAL
;
}
/* MM: This should be 2 for dragonfly plus*/
...
...
@@ -633,12 +620,12 @@ static void dragonfly_read_config(const char *anno, dragonfly_plus_param *params
p
->
num_cn
=
p
->
num_routers
/
2
;
}
rc
=
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_global_c
hannels"
,
anno
,
&
p
->
num_global_channel
s
);
rc
=
configuration_get_value_int
(
&
config
,
"PARAMS"
,
"num_global_c
onnections"
,
anno
,
&
p
->
num_global_connection
s
);
if
(
rc
)
{
printf
(
"
\n
Number of global channels per router not specified, setting to 10 "
);
p
->
num_global_c
hannel
s
=
10
;
p
->
num_global_c
onnection
s
=
10
;
}
p
->
radix
=
p
->
intra_grp_radix
+
p
->
num_global_c
hannel
s
+
p
->
radix
=
p
->
intra_grp_radix
+
p
->
num_global_c
onnection
s
+
p
->
num_cn
;
// TODO this may not be sufficient, radix isn't same for leaf and spine routers
p
->
total_routers
=
p
->
num_groups
*
p
->
num_routers
;
p
->
total_terminals
=
(
p
->
num_groups
*
p
->
num_router_leaf
)
*
p
->
num_cn
;
...
...
@@ -672,6 +659,17 @@ static void dragonfly_read_config(const char *anno, dragonfly_plus_param *params
}
}
// printf("DUMPING INTRA-CONN FOR ROUTER 3:\n");
// printf(" (dest - offset) \n");
// for (int i = 0; i < intraGroupLinks[3].size(); i++)
// {
// for (int j = 0; j < intraGroupLinks[3][i].size(); j++)
// {
// printf(" (%d - %d) ", i, intraGroupLinks[3][i][j].offset);
// }
// }
// printf("\n");
fclose
(
groupFile
);
// read inter group connections, store from a router's perspective
...
...
@@ -818,10 +816,10 @@ void dragonfly_plus_report_stats()
MPI_Reduce
(
&
packet_gen
,
&
total_gen
,
1
,
MPI_LONG
,
MPI_SUM
,
0
,
MPI_COMM_CODES
);
MPI_Reduce
(
&
packet_fin
,
&
total_fin
,
1
,
MPI_LONG
,
MPI_SUM
,
0
,
MPI_COMM_CODES
);
if
(
routing
==
ADAPTIVE
||
routing
==
PROG_ADAPTIVE
)
{
MPI_Reduce
(
&
minimal_count
,
&
total_minimal_packets
,
1
,
MPI_INT
,
MPI_SUM
,
0
,
MPI_COMM_CODES
);
MPI_Reduce
(
&
nonmin_count
,
&
total_nonmin_packets
,
1
,
MPI_INT
,
MPI_SUM
,
0
,
MPI_COMM_CODES
);
}
//
if (routing == ADAPTIVE || routing == PROG_ADAPTIVE) {
//
MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
//
MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
//
}
/* print statistics */
if
(
!
g_tw_mynode
)
{
...
...
@@ -831,11 +829,11 @@ void dragonfly_plus_report_stats()
(
float
)
avg_hops
/
total_finished_chunks
,
avg_time
/
(
total_finished_chunks
*
1000
),
max_time
/
1000
,
(
float
)
final_msg_sz
/
total_finished_msgs
,
total_finished_msgs
,
total_finished_chunks
);
if
(
routing
==
ADAPTIVE
||
routing
==
PROG_ADAPTIVE
)
printf
(
"
\n
ADAPTIVE ROUTING STATS: %d chunks routed minimally %d chunks routed non-minimally "
"completed packets %lld
\n
"
,
total_minimal_packets
,
total_nonmin_packets
,
total_finished_chunks
);
//
if (routing == ADAPTIVE || routing == PROG_ADAPTIVE)
//
printf(
//
"\n ADAPTIVE ROUTING STATS: %d chunks routed minimally %d chunks routed non-minimally "
//
"completed packets %lld \n",
//
total_minimal_packets, total_nonmin_packets, total_finished_chunks);
printf
(
"
\n
Total packets generated %ld finished %ld
\n
"
,
total_gen
,
total_fin
);
}
...
...
@@ -986,42 +984,31 @@ void router_plus_setup(router_state *r, tw_lp *lp)
printf
(
"%lu: %i is a LEAF
\n
"
,
lp
->
gid
,
r
->
router_id
);
}
// setup radix
if
(
r
->
dfp_router_type
==
SPINE
)
{
r
->
inter_radix_util
=
p
->
num_global_channels
;
r
->
intra_radix_util
=
p
->
num_router_leaf
*
p
->
num_level_chans
;
r
->
cn_radix_util
=
0
;
}
else
{
//leaf
r
->
intra_radix_util
=
p
->
num_router_spine
*
p
->
num_level_chans
;
r
->
cn_radix_util
=
p
->
num_cn
;
r
->
inter_radix_util
=
0
;
}
r
->
radix_util
=
r
->
inter_radix_util
+
r
->
intra_radix_util
+
r
->
cn_radix_util
;
r
->
global_channel
=
(
int
*
)
malloc
(
p
->
num_global_channels
*
sizeof
(
int
));
r
->
next_output_available_time
=
(
tw_stime
*
)
malloc
(
r
->
radix_util
*
sizeof
(
tw_stime
));
r
->
link_traffic
=
(
int64_t
*
)
malloc
(
r
->
radix_util
*
sizeof
(
int64_t
));
r
->
link_traffic_sample
=
(
int64_t
*
)
malloc
(
r
->
radix_util
*
sizeof
(
int64_t
));
r
->
vc_occupancy
=
(
int
**
)
malloc
(
r
->
radix_util
*
sizeof
(
int
*
));
r
->
in_send_loop
=
(
int
*
)
malloc
(
r
->
radix_util
*
sizeof
(
int
));
r
->
global_channel
=
(
int
*
)
malloc
(
p
->
num_global_connections
*
sizeof
(
int
));
r
->
next_output_available_time
=
(
tw_stime
*
)
malloc
(
p
->
radix
*
sizeof
(
tw_stime
));
r
->
link_traffic
=
(
int64_t
*
)
malloc
(
p
->
radix
*
sizeof
(
int64_t
));
r
->
link_traffic_sample
=
(
int64_t
*
)
malloc
(
p
->
radix
*
sizeof
(
int64_t
));
r
->
vc_occupancy
=
(
int
**
)
malloc
(
p
->
radix
*
sizeof
(
int
*
));
r
->
in_send_loop
=
(
int
*
)
malloc
(
p
->
radix
*
sizeof
(
int
));
r
->
pending_msgs
=
(
terminal_plus_message_list
***
)
malloc
(
r
->
radix_util
*
sizeof
(
terminal_plus_message_list
**
));
(
terminal_plus_message_list
***
)
malloc
(
p
->
radix
*
sizeof
(
terminal_plus_message_list
**
));
r
->
pending_msgs_tail
=
(
terminal_plus_message_list
***
)
malloc
(
r
->
radix_util
*
sizeof
(
terminal_plus_message_list
**
));
(
terminal_plus_message_list
***
)
malloc
(
p
->
radix
*
sizeof
(
terminal_plus_message_list
**
));
r
->
queued_msgs
=
(
terminal_plus_message_list
***
)
malloc
(
r
->
radix_util
*
sizeof
(
terminal_plus_message_list
**
));
(
terminal_plus_message_list
***
)
malloc
(
p
->
radix
*
sizeof
(
terminal_plus_message_list
**
));
r
->
queued_msgs_tail
=
(
terminal_plus_message_list
***
)
malloc
(
r
->
radix_util
*
sizeof
(
terminal_plus_message_list
**
));
r
->
queued_count
=
(
int
*
)
malloc
(
r
->
radix_util
*
sizeof
(
int
));
r
->
last_buf_full
=
(
tw_stime
**
)
malloc
(
r
->
radix_util
*
sizeof
(
tw_stime
*
));
r
->
busy_time
=
(
tw_stime
*
)
malloc
(
r
->
radix_util
*
sizeof
(
tw_stime
));
r
->
busy_time_sample
=
(
tw_stime
*
)
malloc
(
r
->
radix_util
*
sizeof
(
tw_stime
));
(
terminal_plus_message_list
***
)
malloc
(
p
->
radix
*
sizeof
(
terminal_plus_message_list
**
));
r
->
queued_count
=
(
int
*
)
malloc
(
p
->
radix
*
sizeof
(
int
));
r
->
last_buf_full
=
(
tw_stime
**
)
malloc
(
p
->
radix
*
sizeof
(
tw_stime
*
));
r
->
busy_time
=
(
tw_stime
*
)
malloc
(
p
->
radix
*
sizeof
(
tw_stime
));
r
->
busy_time_sample
=
(
tw_stime
*
)
malloc
(
p
->
radix
*
sizeof
(
tw_stime
));
rc_stack_create
(
&
r
->
st
);
for
(
int
i
=
0
;
i
<
r
->
radix_util
;
i
++
)
{
for
(
int
i
=
0
;
i
<
p
->
radix
;
i
++
)
{
// Set credit & router occupancy
r
->
busy_time
[
i
]
=
0
.
0
;
r
->
busy_time_sample
[
i
]
=
0
.
0
;
...
...
@@ -1471,8 +1458,8 @@ static void packet_arrive_rc(terminal_state *s, tw_bf *bf, terminal_plus_message
tw_rand_reverse_unif
(
lp
->
rng
);
if
(
msg
->
path_type
==
MINIMAL
)
minimal_count
--
;
if
(
msg
->
path_type
==
NON_MINIMAL
)
nonmin_count
--
;
//
if (msg->path_type == NON_MINIMAL)
//
nonmin_count--;
N_finished_chunks
--
;
s
->
finished_chunks
--
;
...
...
@@ -1649,15 +1636,15 @@ static void packet_arrive(terminal_state *s, tw_bf *bf, terminal_plus_message *m
if
(
msg
->
path_type
==
MINIMAL
)
minimal_count
++
;
if
(
msg
->
path_type
==
NON_MINIMAL
)
nonmin_count
++
;
//
if (msg->path_type == NON_MINIMAL)
//
nonmin_count++;
if
(
msg
->
chunk_id
==
num_chunks
-
1
)
{
bf
->
c31
=
1
;
s
->
packet_fin
++
;
packet_fin
++
;
}
if
(
msg
->
path_type
!=
MINIMAL
&&
msg
->
path_type
!=
NON_MINIMAL
)
if
(
msg
->
path_type
!=
MINIMAL
)
printf
(
"
\n
Wrong message path type %d "
,
msg
->
path_type
);
/* save the sample time */
...
...
@@ -1775,13 +1762,13 @@ void dragonfly_plus_rsample_init(router_state *s, tw_lp *lp)
int
i
=
0
;
const
dragonfly_plus_param
*
p
=
s
->
params
;
assert
(
s
->
radix_util
);
assert
(
p
->
radix
);
s
->
max_arr_size
=
MAX_STATS
;
s
->
rsamples
=
(
struct
dfly_router_sample
*
)
malloc
(
MAX_STATS
*
sizeof
(
struct
dfly_router_sample
));
for
(;
i
<
s
->
max_arr_size
;
i
++
)
{
s
->
rsamples
[
i
].
busy_time
=
(
tw_stime
*
)
malloc
(
sizeof
(
tw_stime
)
*
s
->
radix_util
);
s
->
rsamples
[
i
].
link_traffic_sample
=
(
int64_t
*
)
malloc
(
sizeof
(
int64_t
)
*
s
->
radix_util
);
s
->
rsamples
[
i
].
busy_time
=
(
tw_stime
*
)
malloc
(
sizeof
(
tw_stime
)
*
p
->
radix
);
s
->
rsamples
[
i
].
link_traffic_sample
=
(
int64_t
*
)
malloc
(
sizeof
(
int64_t
)
*
p
->
radix
);
}
}
void
dragonfly_plus_rsample_rc_fn
(
router_state
*
s
,
tw_bf
*
bf
,
terminal_plus_message
*
msg
,
tw_lp
*
lp
)
...
...
@@ -1797,12 +1784,12 @@ void dragonfly_plus_rsample_rc_fn(router_state *s, tw_bf *bf, terminal_plus_mess
const
dragonfly_plus_param
*
p
=
s
->
params
;
int
i
=
0
;
for
(;
i
<
s
->
radix_util
;
i
++
)
{
for
(;
i
<
p
->
radix
;
i
++
)
{
s
->
busy_time_sample
[
i
]
=
stat
.
busy_time
[
i
];
s
->
link_traffic_sample
[
i
]
=
stat
.
link_traffic_sample
[
i
];
}
for
(
i
=
0
;
i
<
s
->
radix_util
;
i
++
)
{
for
(
i
=
0
;
i
<
p
->
radix
;
i
++
)
{
stat
.
busy_time
[
i
]
=
0
;
stat
.
link_traffic_sample
[
i
]
=
0
;
}
...
...
@@ -1835,7 +1822,7 @@ void dragonfly_plus_rsample_fn(router_state *s, tw_bf *bf, terminal_plus_message
s
->
rsamples
[
cur_indx
].
fwd_events
=
s
->
fwd_events
;
s
->
rsamples
[
cur_indx
].
rev_events
=
s
->
rev_events
;
for
(;
i
<
s
->
radix_util
;
i
++
)
{
for
(;
i
<
p
->
radix
;
i
++
)
{
s
->
rsamples
[
cur_indx
].
busy_time
[
i
]
=
s
->
busy_time_sample
[
i
];
s
->
rsamples
[
cur_indx
].
link_traffic_sample
[
i
]
=
s
->
link_traffic_sample
[
i
];
}
...
...
@@ -1846,7 +1833,7 @@ void dragonfly_plus_rsample_fn(router_state *s, tw_bf *bf, terminal_plus_message
s
->
fwd_events
=
0
;
s
->
rev_events
=
0
;
for
(
i
=
0
;
i
<
s
->
radix_util
;
i
++
)
{
for
(
i
=
0
;
i
<
p
->
radix
;
i
++
)
{
s
->
busy_time_sample
[
i
]
=
0
;
s
->
link_traffic_sample
[
i
]
=
0
;
}
...
...
@@ -1868,7 +1855,7 @@ void dragonfly_plus_rsample_fin(router_state *s, tw_lp *lp)
"(double)
\n
"
"link traffic for each of the %d links (int64_t)
\n
sample end time (double) forward events "
"per sample
\n
reverse events per sample "
,
s
->
radix_util
,
s
->
radix_util
);
p
->
radix
,
p
->
radix
);
fclose
(
fp
);
}
char
rt_fn
[
MAX_NAME_LENGTH
];
...
...
@@ -1879,15 +1866,15 @@ void dragonfly_plus_rsample_fin(router_state *s, tw_lp *lp)
int
i
=
0
;
int
size_sample
=
sizeof
(
tw_lpid
)
+
s
->
radix_util
*
(
sizeof
(
int64_t
)
+
sizeof
(
tw_stime
))
+
int
size_sample
=
sizeof
(
tw_lpid
)
+
p
->
radix
*
(
sizeof
(
int64_t
)
+
sizeof
(
tw_stime
))
+
sizeof
(
tw_stime
)
+
2
*
sizeof
(
long
);
FILE
*
fp
=
fopen
(
rt_fn
,
"a"
);
fseek
(
fp
,
sample_rtr_bytes_written
,
SEEK_SET
);
for
(;
i
<
s
->
op_arr_size
;
i
++
)
{
fwrite
((
void
*
)
&
(
s
->
rsamples
[
i
].
router_id
),
sizeof
(
tw_lpid
),
1
,
fp
);
fwrite
(
s
->
rsamples
[
i
].
busy_time
,
sizeof
(
tw_stime
),
s
->
radix_util
,
fp
);
fwrite
(
s
->
rsamples
[
i
].
link_traffic_sample
,
sizeof
(
int64_t
),
s
->
radix_util
,
fp
);
fwrite
(
s
->
rsamples
[
i
].
busy_time
,
sizeof
(
tw_stime
),
p
->
radix
,
fp
);
fwrite
(
s
->
rsamples
[
i
].
link_traffic_sample
,
sizeof
(
int64_t
),
p
->
radix
,
fp
);
fwrite
((
void
*
)
&
(
s
->
rsamples
[
i
].
end_time
),
sizeof
(
tw_stime
),
1
,
fp
);
fwrite
((
void
*
)
&
(
s
->
rsamples
[
i
].
fwd_events
),
sizeof
(
long
),
1
,
fp
);
fwrite
((
void
*
)
&
(
s
->
rsamples
[
i
].
rev_events
),
sizeof
(
long
),
1
,
fp
);
...
...
@@ -2113,7 +2100,7 @@ void dragonfly_plus_router_final(router_state *s, tw_lp *lp)
{
free
(
s
->
global_channel
);
int
i
,
j
;
for
(
i
=
0
;
i
<
s
->
radix_util
;
i
++
)
{
for
(
i
=
0
;
i
<
s
->
params
->
radix
;
i
++
)
{
for
(
j
=
0
;
j
<
s
->
params
->
num_vcs
;
j
++
)
{
if
(
s
->
queued_msgs
[
i
][
j
]
!=
NULL
)
{
printf
(
"[%llu] leftover queued messages %d %d %d
\n
"
,
LLU
(
lp
->
gid
),
i
,
j
,
...
...
@@ -2139,7 +2126,7 @@ void dragonfly_plus_router_final(router_state *s, tw_lp *lp)
}
written
+=
sprintf
(
s
->
output_buf
+
written
,
"
\n
%llu %d %d"
,
LLU
(
lp
->
gid
),
s
->
router_id
/
p
->
num_routers
,
s
->
router_id
%
p
->
num_routers
);
for
(
int
d
=
0
;
d
<
s
->
radix_util
;
d
++
)
for
(
int
d
=
0
;
d
<
p
->
radix
;
d
++
)
written
+=
sprintf
(
s
->
output_buf
+
written
,
" %lf"
,
s
->
busy_time
[
d
]);
sprintf
(
s
->
output_buf
+
written
,
"
\n
"
);
...
...
@@ -2154,7 +2141,7 @@ void dragonfly_plus_router_final(router_state *s, tw_lp *lp)
written
+=
sprintf
(
s
->
output_buf2
+
written
,
"
\n
%llu %d %d"
,
LLU
(
lp
->
gid
),
s
->
router_id
/
p
->
num_routers
,
s
->
router_id
%
p
->
num_routers
);
for
(
int
d
=
0
;
d
<
s
->
radix_util
;
d
++
)
for
(
int
d
=
0
;
d
<
p
->
radix
;
d
++
)
written
+=
sprintf
(
s
->
output_buf2
+
written
,
" %lld"
,
LLD
(
s
->
link_traffic
[
d
]));
lp_io_write
(
lp
->
gid
,
(
char
*
)
"dragonfly-plus-router-traffic"
,
written
,
s
->
output_buf2
);
...
...
@@ -2165,11 +2152,11 @@ void dragonfly_plus_router_final(router_state *s, tw_lp *lp)
* leaf router alone. */
/* Neil: For now I think this will work as is. With a 2 level fat tree at the group level,
* this is overkill, perhaps.
* this is overkill, perhaps.
**/
static
vector
<
int
>
get_intra_router
(
router_state
*
s
,
int
src_router_id
,
int
dest_router_id
,
int
num_rtrs_per_grp
)
static
vector
<
int
>
get_intra_router
(
router_state
*
s
,
int
src_router_id
,
int
dest_router_id
)
{
int
num_rtrs_per_grp
=
s
->
params
->
num_routers
;
/* Check for intra-group connections */
int
src_rel_id
=
src_router_id
%
num_rtrs_per_grp
;
int
dest_rel_id
=
dest_router_id
%
num_rtrs_per_grp
;
...
...
@@ -2227,7 +2214,7 @@ static tw_lpid get_next_stop(router_state *s,
int
origin_grp_id
=
msg
->
origin_router_id
/
s
->
params
->
num_routers
;
int
select_chan
=
-
1
;
/* If the packet has arrived at the destination router -------------------------------*/
if
(
dest_router_id
==
local_router_id
)
{
dest_lp
=
msg
->
dest_terminal_id
;
...
...
@@ -2237,8 +2224,14 @@ static tw_lpid get_next_stop(router_state *s,
/* If the packet has arrived at the destination group --------------------------------*/
if
(
s
->
group_id
==
dest_group_id
)
{
bf
->
c19
=
1
;
vector
<
int
>
next_stop
=
get_intra_router
(
s
,
local_router_id
,
dest_router_id
,
s
->
params
->
num_routers
);
bf
->
c19
=
1
;
vector
<
int
>
next_stop
=
get_intra_router
(
s
,
local_router_id
,
dest_router_id
);
printf
(
"%d: Router next stop is: ["
,
s
->
router_id
);
for
(
vector
<
int
>::
iterator
it
=
next_stop
.
begin
();
it
!=
next_stop
.
end
();
it
++
)
{
printf
(
" %d "
,
*
it
);
}
printf
(
"]
\n
"
);
assert
(
!
next_stop
.
empty
());
select_chan
=
tw_rand_integer
(
lp
->
rng
,
0
,
next_stop
.
size
()
-
1
);
...
...
@@ -2258,6 +2251,8 @@ static tw_lpid get_next_stop(router_state *s,
if
(
msg
->
last_hop
==
TERMINAL
)
{
assert
(
s
->
dfp_router_type
==
LEAF
);
//We're a leaf
//Need to select a spine router from current group to forward to
bf
->
c19
=
1
;
select_chan
=
tw_rand_integer
(
lp
->
rng
,
0
,
connectionList
[
my_grp_id
][
dest_group_id
].
size
()
-
1
);
...
...
@@ -2285,145 +2280,90 @@ static tw_lpid get_next_stop(router_state *s,
/* gets the output port corresponding to the next stop of the message */
static
int
get_output_port
(
router_state
*
s
,
terminal_plus_message
*
msg
,
tw_lp
*
lp
,
tw_bf
*
bf
,
tw_lpid
next_stop
)
{
// tw_error(TW_LOC, "\n DRAGONFLY PLUS GET OUTPUT PORT NOT IMPLEMENTED");
printf
(
"%lu, Router %d: Get Output Port()
\n
"
,
lp
->
gid
,
s
->
router_id
);
printf
(
"%d: Get Output Port()
\n
"
,
s
->
router_id
);
int
output_port
=
-
1
;
const
dragonfly_plus_param
*
p
=
s
->
params
;
int
output_port
=
0
;
int
rand_offset
=
-
1
;
int
terminal_id
=
codes_mapping_get_lp_relative_id
(
msg
->
dest_terminal_id
,
0
,
0
);
const
dragonfly_plus_param
*
p
=
s
->
params
;
int
dest_router
=
codes_mapping_get_lp_relative_id
(
next_stop
,
0
,
0
);
//turns lpid into the corresponding router id
int
dest_group_id
=
dest_router
/
p
->
num_routers
;
int
src_router
=
s
->
router_id
;
printf
(
"
\t
dest_router = %d (%lu)
\n
"
,
dest_router
,
next_stop
);
if
((
tw_lpid
)
next_stop
==
msg
->
dest_terminal_id
)
{
// if the next stop is the destination
/* Make a random number selection (only for reverse computation) */
int
rand_sel
=
tw_rand_integer
(
lp
->
rng
,
0
,
terminal_id
);
assert
(
s
->
dfp_router_type
==
LEAF
);
if
(
s
->
dfp_router_type
==
LEAF
)
{
//packet from leaf must go to either destination terminal or to a spine router
output_port
=
s
->
intra_radix_util
+
s
->
inter_radix_util
+
(
terminal_id
%
s
->
cn_radix_util
);
printf
(
"
\t
%d
\n
"
,
output_port
);
// output_port = p->intra_grp_radix + p->num_global_channels + (terminal_id % p->num_cn);
if
((
tw_lpid
)
next_stop
==
msg
->
dest_terminal_id
)
{
// if the next stop is the destination
int
rand_sel
=
tw_rand_integer
(
lp
->
rng
,
0
,
2
);
// Make a random number selection (only for reverse computation)
output_port
=
p
->
intra_grp_radix
+
p
->
num_global_connections
+
(
terminal_id
%
p
->
num_cn
);
//TODO this seems wrong?
printf
(
"
\t
Port: %d (terminal)
\n
"
,
output_port
);
}
else
{
// next stop is not the destination terminal - must go to spine router in same group
int
src_loc_id
=
src_router
%
p
->
num_routers
;
int
dest_loc_id
=
dest_router
%
p
->
num_routers
;
int
num_links
=
intraGroupLinks
[
src_loc_id
][
dest_loc_id
].
size
();
//number of ports connecting src_router to dest_router
printf
(
"
\t
num_links to dest = %d
\n
"
,
num_links
);
if
(
routing
==
MINIMAL
)
{
//minimal doesn't care about adaptive routing, ARNs. Choose port to dest at random
int
rand_sel
=
tw_rand_integer
(
lp
->
rng
,
0
,
num_links
-
1
);
int
offset
=
intraGroupLinks
[
src_loc_id
][
dest_loc_id
][
rand_sel
].
offset
;
output_port
=
offset
;
//first ports are the intra-group ports so offset alone is sufficient
printf
(
"
\t
Port: %d (intra)
\n
"
,
output_port
);
}
else
{
tw_error
(
TW_LOC
,
"
\n
get_output_port() Unsupported Routing Algorithm"
);
}
}
}
// else {
// int intm_grp_id = dest_router / p->num_routers;
// int rand_offset = -1;
// if (intm_grp_id != s->group_id) {
// /* traversing a global channel */
// vector< bLink > &curVec = interGroupLinks[src_router][intm_grp_id];
// if (interGroupLinks[src_router][intm_grp_id].size() == 0)
// printf("\n Source router %d intm_grp_id %d ", src_router, intm_grp_id);
// assert(interGroupLinks[src_router][intm_grp_id].size() > 0);
// rand_offset = tw_rand_integer(lp->rng, 0, interGroupLinks[src_router][intm_grp_id].size() - 1);
// assert(rand_offset >= 0);
// bLink bl = interGroupLinks[src_router][intm_grp_id][rand_offset];
// int channel_id = bl.offset;
// output_port = p->intra_grp_radix + channel_id;
// }
// else {
// int intra_rtr_id = (dest_router % p->num_routers);
// int intragrp_rtr_id = s->router_id % p->num_routers;
// int src_col = intragrp_rtr_id % p->num_router_cols;
// int src_row = intragrp_rtr_id / p->num_router_cols;
// int dest_col = intra_rtr_id % p->num_router_cols;
// int dest_row = intra_rtr_id / p->num_router_cols;
// if (src_col == dest_col) {
// int offset = tw_rand_integer(lp->rng, 0, p->num_col_chans - 1);
// output_port = p->num_router_cols * p->num_row_chans + dest_row * p->num_col_chans + offset;
// assert(output_port < p->intra_grp_radix);
// }
// else {
// if (src_row == dest_row) {
// int offset = tw_rand_integer(lp->rng, 0, p->num_row_chans - 1);
// output_port = dest_col * p->num_row_chans + offset;
// assert(output_port < (s->params->num_router_cols * p->num_row_chans));
// }
// else {
// tw_error(TW_LOC,
// "\n Invalid dragonfly connectivity src row %d dest row %d src col %d dest col "
// "%d src %d dest %d",
// src_row, dest_row, src_col, dest_col, intragrp_rtr_id, intra_rtr_id);
// }
// }
// }
// }
return
output_port
;
}
/* MM: This will no longer be needed for dragonfly plus. We will have to
* implement progressive adaptive routing though. */
static
void
do_local_adaptive_routing
(
router_state
*
s
,
tw_lp
*
lp
,
terminal_plus_message
*
msg
,
tw_bf
*
bf
,
int
dest_router_id
,
int
intm_router_id
)
{
tw_lpid
min_rtr_id
,
nonmin_rtr_id
;
int
min_port
,
nonmin_port
;
else
if
(
s
->
dfp_router_type
==
SPINE
)
{
//packet from spine must go to either a leaf in the same group or to a spine router in another group
if
(
dest_group_id
==
s
->
group_id
)
{
// if the destination group id is this routers group
//needs to send to leaf via intra group connections
int
src_loc_id
=
src_router
%
p
->
num_routers
;
int
dest_loc_id
=
dest_router
%
p
->
num_routers
;
int
dest_grp_id
=
dest_router_id
/
s
->
params
->
num_routers
;
int
intm_grp_id
=
intm_router_id
/
s
->
params
->
num_routers
;
int
my_grp_id
=
s
->
router_id
/
s
->
params
->
num_routers
;
int
num_links
=
intraGroupLinks
[
src_loc_id
][
dest_loc_id
].
size
();
//number of ports connecting src_router to dest_router
if
(
my_grp_id
!=
dest_grp_id
||
my_grp_id
!=
intm_grp_id
)
tw_error
(
TW_LOC
,
"
\n
Invalid local routing my grp id %d dest_gid %d intm_gid %d intm rid %d"
,
my_grp_id
,
dest_grp_id
,
intm_grp_id
,
intm_router_id
);
if
(
routing
==
MINIMAL
)
{
//minimal doesn't care about adaptive routing, ARNs. Choose port to dest at random
int
rand_sel
=
tw_rand_integer
(
lp
->
rng
,
0
,
num_links
-
1
);
int
offset
=
intraGroupLinks
[
src_loc_id
][
dest_loc_id
][
rand_sel
].
offset
;
output_port
=
offset
;
//first ports are the intra-group ports so offset alone is sufficient
printf
(
"
\t
Port: %d (intra)
\n
"
,
output_port
);
int
min_chan
=
-
1
,
nonmin_chan
=
-
1
;
vector
<
int
>
next_min_stops
=
get_intra_router
(
s
,
s
->
router_id
,
dest_router_id
,
s
->
params
->
num_routers
);
vector
<
int
>
next_nonmin_stops
=
get_intra_router
(
s
,
s
->
router_id
,
intm_router_id
,
s
->
params
->
num_routers
);
min_chan
=
tw_rand_integer
(
lp
->
rng
,
0
,
next_min_stops
.
size
()
-
1
);
nonmin_chan
=
tw_rand_integer
(
lp
->
rng
,
0
,
next_nonmin_stops
.
size
()
-
1
);
codes_mapping_get_lp_id
(
lp_group_name
,
LP_CONFIG_NM_ROUT
,
s
->
anno
,
0
,
next_min_stops
[
min_chan
]
/
num_routers_per_mgrp
,
next_min_stops
[
min_chan
]
%
num_routers_per_mgrp
,
&
min_rtr_id
);
codes_mapping_get_lp_id
(
lp_group_name
,
LP_CONFIG_NM_ROUT
,
s
->
anno
,
0
,
next_nonmin_stops
[
nonmin_chan
]
/
num_routers_per_mgrp
,
next_nonmin_stops
[
nonmin_chan
]
%
num_routers_per_mgrp
,
&
nonmin_rtr_id
);
min_port
=
get_output_port
(
s
,
msg
,
lp
,
bf
,
min_rtr_id
);
nonmin_port
=
get_output_port
(
s
,
msg
,
lp
,
bf
,
nonmin_rtr_id
);
int
min_port_count
=
0
,
nonmin_port_count
=
0
;
for
(
int
k
=
0
;
k
<
s
->
params
->
num_vcs
;
k
++
)
min_port_count
+=
s
->
vc_occupancy
[
min_port
][
k
];
min_port_count
+=
s
->
queued_count
[
min_port
];
for
(
int
k
=
0
;
k
<
s
->
params
->
num_vcs
;
k
++
)
nonmin_port_count
+=
s
->
vc_occupancy
[
nonmin_port
][
k
];
nonmin_port_count
+=
s
->
queued_count
[
nonmin_port
];
}
else
{
tw_error
(
TW_LOC
,
"
\n
get_output_port() Unsupported Routing Algorithm"
);
}
}
else
{
// the next stop is a spine router in a different group
int
local_stop
=
-
1
;
tw_lpid
global_stop
;
int
num_links
=
interGroupLinks
[
src_router
][
dest_group_id
].
size
();
//TODO - THIS IS A PROBLEM RIGHT NOW
printf
(
"
\t
num_links (inter) to dest = %d"
,
num_links
);
if
(
routing
==
MINIMAL
)
{
int
rand_sel
=
tw_rand_integer
(
lp
->
rng
,
0
,
num_links
-
1
);
int
offset
=
interGroupLinks
[
src_router
][
dest_group_id
][
rand_sel
].
offset
;
//offset of global connections
output_port
=
p
->
intra_grp_radix
+
offset
;
printf
(
"
\t
Port: %d (inter)
\n
"
,
output_port
);
// if(nonmin_port_count * num_intra_nonmin_hops > min_port_count * num_intra_min_hops)
if
(
nonmin_port_count
>
min_port_count
)
{
msg
->
path_type
=
MINIMAL
;
}
else
{
tw_error
(
TW_LOC
,
"
\n
get_output_port() Unsupported Routing Algorithm"
);
}
}
}
else
{
// printf("\n Nonmin port count %ld min port count %ld ", nonmin_port_count, min_port_count);
msg
->
path_type
=
NON_MINIMAL
;
tw_error
(
TW_LOC
,
"
\n
get_output_port() Undefined router type found for gid %d"
,
lp
->
gid
);
}
return
output_port
;
}
/*MM: Change this to do_adaptive_routing. Do you know what are the congestion sensing mechanism for adaptive
* routing in dragonfly plus? Is it similar to dragonfly? */
//Neil: Changed name, nothing else yet.
...
...
@@ -2438,210 +2378,210 @@ static int do_adaptive_routing(router_state *s,
// decide which routing to take
// get the queue occupancy of both the minimal and non-minimal output ports
int
dest_grp_id
=
dest_router_id
/
s
->
params
->
num_routers
;
int
intm_grp_id
=
intm_id
/
s
->
params
->
num_routers
;
int
my_grp_id
=
s
->
router_id
/
s
->
params
->
num_routers
;
int
num_min_chans
=
connectionList
[
my_grp_id
][
dest_grp_id
].
size
();
int
num_nonmin_chans
=
connectionList
[
my_grp_id
][
intm_grp_id
].
size
();
int
min_chan_a
,
min_chan_b
,
nonmin_chan_a
,
nonmin_chan_b
;
int
min_rtr_a
,
min_rtr_b
,
nonmin_rtr_a
,
nonmin_rtr_b
;
vector
<
int
>
dest_rtr_as
,
dest_rtr_bs
;
int
min_port_a
,
min_port_b
,
nonmin_port_a
,
nonmin_port_b
;
tw_lpid
min_rtr_a_id
,
min_rtr_b_id
,
nonmin_rtr_a_id
,
nonmin_rtr_b_id
;
bool
noIntraA
,
noIntraB
;
/* two possible routes to minimal destination */
min_chan_a
=
tw_rand_integer
(
lp
->
rng
,
0
,
num_min_chans
-
1
);
min_chan_b
=
tw_rand_integer
(
lp
->
rng
,
0
,
num_min_chans
-
1
);
if
(
min_chan_a
==
min_chan_b
&&
num_min_chans
>
1
)
min_chan_b
=
(
min_chan_a
+
1
)
%
num_min_chans
;
int
chana1
=
0
;
// chana1 = tw_rand_integer(lp->rng, 0, interGroupLinks[s->router_id][dest_grp_id].size()-1);
// chana1=0;
min_rtr_a
=
connectionList
[
my_grp_id
][
dest_grp_id
][
min_chan_a
];
noIntraA
=
false
;
if
(
min_rtr_a
==
s
->
router_id
)
{
noIntraA
=
true
;
min_rtr_a
=
interGroupLinks
[
s
->
router_id
][
dest_grp_id
][
chana1
].
dest
;
}
if
(
num_min_chans
>
1
)
{
noIntraB
=
false
;
min_rtr_b
=
connectionList
[
my_grp_id
][
dest_grp_id
][
min_chan_b
];
if
(
min_rtr_b
==
s
->
router_id
)
{
noIntraB
=
true
;
min_rtr_b
=
interGroupLinks
[
s
->
router_id
][
dest_grp_id
][
chana1
].
dest
;
}
}
if
(
noIntraA
)
{
dest_rtr_as
.
push_back
(
min_rtr_a
);
}
else
{
dest_rtr_as
=
get_intra_router
(
s
,
s
->
router_id
,
min_rtr_a
,
s
->
params
->
num_routers
);
}
int
dest_rtr_b_sel
;
int
dest_rtr_a_sel
=
tw_rand_integer
(
lp
->
rng
,
0
,
dest_rtr_as
.
size
()
-
1
);