diff --git a/codes/connection-manager.h b/codes/connection-manager.h index 64e6b86860686194b52bdc0fac8293292ba09a07..91c6d2720815c6c6b286ae13f915c370bab474a5 100644 --- a/codes/connection-manager.h +++ b/codes/connection-manager.h @@ -43,6 +43,11 @@ struct Connection ConnectionType conn_type; //type of the connection: CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL }; +inline bool operator<(const Connection& lhs, const Connection& rhs) +{ + return lhs.port < rhs.port; +} + /** * @class ConnectionManager * diff --git a/src/networks/model-net/dragonfly-plus.C b/src/networks/model-net/dragonfly-plus.C index b6c0a2e03cec1aad2b86fcecb418bbdd0652825d..415e1f852c14f42a4ebde79fe3813a3d9d6c351d 100644 --- a/src/networks/model-net/dragonfly-plus.C +++ b/src/networks/model-net/dragonfly-plus.C @@ -157,6 +157,21 @@ static void delete_terminal_plus_message_list(void *thisO) free(toDel); } +template +vector< T > set_difference_vectors(vector vec1, vector vec2) +{ + int max_len = std::max(vec1.size(), vec2.size()); + vector< T > retVec(max_len); + typename vector< T >::iterator retIt; + + retIt = std::set_difference(vec1.begin(), vec1.end(), vec2.begin(), vec2.end(), retVec.begin()); + + retVec.resize(retIt - retVec.begin()); + + return retVec; +} + + struct dragonfly_plus_param { // configuration parameters @@ -608,7 +623,7 @@ void dragonfly_plus_print_params(const dragonfly_plus_param *p) printf("\tnum_router_spine = %d\n",p->num_router_spine); printf("\tnum_router_leaf = %d\n",p->num_router_leaf); printf("\tadaptive_threshold = %d\n",p->adaptive_threshold); - printf("\tmax_port_score = %d\n",p->max_port_score); + printf("\tmax_port_score = %ld\n",p->max_port_score); printf("\tnum_groups = %d\n",p->num_groups); printf("\tradix = %d\n",p->radix); printf("\ttotal_routers = %d\n",p->total_routers); @@ -2667,7 +2682,7 @@ static int dfp_pick_intermediate_router_with_conn(router_state *s, tw_bf *bf, te return intm_router_id; } -static int dfp_reselect_intermediate_router(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, int origin_group_id, int fdest_group_id) +static void dfp_reselect_intermediate_router(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, int origin_group_id, int fdest_group_id) { assert(s->params->source_group_rerouting == 0); //this method should only be called if rerouting isn't allowed if (s->params->intermediate_router_choice == INT_CHOICE_SPINE) { //then we care about having a connection to the actual spine router @@ -2858,72 +2873,80 @@ static Connection get_best_connection_from_conns(router_state *s, tw_bf *bf, ter return selected_conns[best_score_index]; } -static vector< Connection > get_possible_stops_to_specific_router(router_state *s, - tw_bf *bf, - terminal_plus_message *msg, - tw_lp *lp, - int specific_router_id) + +//Returns a vector of connections that are legal dragonfly plus routes that specifically would not allow for a minimal connection to the specific router specified in get_possible_stops_to_specific_router() +//Be very wary of using this method, results may not make sense if possible_minimal_stops is not a vector of minimal next stops to fdest_rotuer_id +//Nonminimal specifically refers to any move that does not move directly toward the destination router +static vector< Connection > get_legal_nonminimal_stops(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, vector< Connection > possible_minimal_stops, int fdest_router_id) { int my_router_id = s->router_id; int my_group_id = s->router_id / s->params->num_routers; - int specific_group_id = specific_router_id / s->params->num_routers; - router_type specific_router_type = router_type_map[specific_router_id]; + int origin_group_id = msg->origin_router_id / s->params->num_routers; + int fdest_group_id = fdest_router_id / s->params->num_routers; + bool in_intermediate_group = (my_group_id != origin_group_id) && (my_group_id != fdest_group_id); - // if (specific_router_type == SPINE) - // tw_error(TW_LOC, "ROUTING TO SPINE NOT SUPPORTED\n"); //TODO when picking intm_rtr_id at the origin leaf, we need to pick a spine router that has a direct connection to our group! - vector< Connection > possible_next_conns; + vector< Connection > possible_nonminimal_stops; - if (my_router_id == specific_router_id) { - return possible_next_conns; //we're there so theres no need to go further. return empty - routing should check if we've arrived before calling this method + if (my_group_id == origin_group_id) { //then we're just sending upward out of the group + if (s->dfp_router_type == LEAF) { //then any connections to spines that are not in possible_minimal_stops should be included + vector< Connection > conns_to_spines = s->connMan->get_connections_by_type(CONN_LOCAL); + possible_nonminimal_stops = set_difference_vectors(conns_to_spines, possible_minimal_stops); //get the complement of possible_minimal_stops + } + else if (s->dfp_router_type == SPINE) { //then we have to send via global connections that aren't to the dest group + vector< Connection > conns_to_other_groups = s->connMan->get_connections_by_type(CONN_GLOBAL); + possible_nonminimal_stops = set_difference_vectors(conns_to_other_groups, possible_minimal_stops); + } } - else if (my_group_id == specific_group_id) { - - if (s->dfp_router_type == SPINE) { //we are spine and we are not the specific router - if (specific_router_type == SPINE) { //the specific router is a spine and we're not it so then we need to send to one of our leafs first on local - possible_next_conns = s->connMan->get_connections_by_type(CONN_LOCAL); - } - else { // Then the spec router's a leaf and we have a local connection to the specific router on local - assert(specific_router_type == LEAF); - possible_next_conns = s->connMan->get_connections_to_gid(specific_router_id, CONN_LOCAL); - } + else if (in_intermediate_group) { + if (msg->dfp_upward_channel_flag == 1) { //then we return an empty vector as the only legal moves are those that already exist in possible_minimal_stops + assert(possible_nonminimal_stops.size() == 0); + return possible_nonminimal_stops; } - else { //we're a leaf in the specific router group - assert(s->dfp_router_type == LEAF); - if (specific_router_type == SPINE) { //Then we have a local connection to the specific router on local - possible_next_conns = s->connMan->get_connections_to_gid(specific_router_id, CONN_LOCAL); - } - else { //then the specific router is a leaf within our group and we need to send to one of our spines first - assert(specific_router_type == LEAF); - possible_next_conns = s->connMan->get_connections_by_type(CONN_LOCAL); - } + else if (s->dfp_router_type == LEAF) { //if we're a leaf in an intermediate group then the routing alg should have flipped the dfp_upward channel already but lets return empty just in case + assert(possible_nonminimal_stops.size() == 0); + return possible_nonminimal_stops; + } + else if (s->dfp_router_type == SPINE) { //then possible_minimal_stops will be the list of connections + assert(msg->dfp_upward_channel_flag == 0); + vector< Connection> conns_to_leaves = s->connMan->get_connections_by_type(CONN_LOCAL); + possible_nonminimal_stops = set_difference_vectors(conns_to_leaves, possible_minimal_stops); //get the complement of possible_minimal_stops + } + else { + tw_error(TW_LOC, "Impossible Error - Something's majorly wrong if this is tripped"); } + + } + else if (my_group_id == fdest_group_id) { + assert(possible_nonminimal_stops.size() == 0); //empty because all legal stops within the destination group are minimal + return possible_nonminimal_stops; + } + else { + tw_error(TW_LOC, "Invalid group classification\n"); } - else { //then we are not the specific router, nor are we in the specific router's group. - if (s->dfp_router_type == SPINE) { //then we need to send to the specific group if we have a connection, otherwise we send to a leaf in our group - // if (specific_router_type == SPINE) { //then it's not good enough to just send to the specific group, we have to send to the specific router - // vector< Connection > conns_to_spec_router = s->connMan->get_connections_to_gid(specific_router_id, CONN_GLOBAL); - // if (conns_to_spec_router.size() < 1) - // tw_error(TW_LOC, "Failed to find a connection to specific router\n"); - // possible_next_conns = conns_to_spec_router; - // } - - vector< Connection > conns_to_spec_group = s->connMan->get_connections_to_group(specific_group_id); - if (conns_to_spec_group.size() < 1) { //then we have to send to a leaf on local - possible_next_conns = s->connMan->get_connections_by_type(CONN_LOCAL); - } - else { //then we can send to the specific group via global conn - possible_next_conns = conns_to_spec_group; - } - } - else { //then we are a leaf and need to send to a spine that has a connection to the spec router group - assert(s->dfp_router_type == LEAF); + // assert(possible_nonminimal_stops.size() > 0); + return possible_nonminimal_stops; +} + +//The term minimal in this case refers to "Moving toward destination router". If a packet is at an intermediate spine and that spine doesn't +//have a direct connection to the destinatino group, then there wouldn't be any "legal minimal stops". The packet would have to continue to +//some leaf in the group first. THIS DOES NOT INCLUDE REROUTING. The only time an intermediate leaf can send to an intermediate spine is if +//dfp_upward_channel_flag is 0. +static vector< Connection > get_legal_minimal_stops(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, int fdest_router_id) +{ + int my_router_id = s->router_id; + int my_group_id = s->router_id / s->params->num_routers; + int origin_group_id = msg->origin_router_id / s->params->num_routers; + int fdest_group_id = fdest_router_id / s->params->num_routers; + + if (my_group_id != fdest_group_id) { //then we're in the source or the intermediate group. + if (s->dfp_router_type == LEAF) { vector< Connection> possible_next_conns_to_group; set poss_router_id_set_to_group; - for(int i = 0; i < connectionList[my_group_id][specific_group_id].size(); i++) + for(int i = 0; i < connectionList[my_group_id][fdest_group_id].size(); i++) { - int poss_router_id = connectionList[my_group_id][specific_group_id][i]; + int poss_router_id = connectionList[my_group_id][fdest_group_id][i]; // printf("%d\n",poss_router_id); if (poss_router_id_set_to_group.count(poss_router_id) == 0) { //if we haven't added the connections from poss_router_id yet vector< Connection > conns = s->connMan->get_connections_to_gid(poss_router_id, CONN_LOCAL); @@ -2931,36 +2954,37 @@ static vector< Connection > get_possible_stops_to_specific_router(router_state * possible_next_conns_to_group.insert(possible_next_conns_to_group.end(), conns.begin(), conns.end()); } } - if (possible_next_conns_to_group.size() < 1) - tw_error(TW_LOC, "Something went wrong when trying to send to a spine with connection to spec group\n"); - - // if (specific_router_type == SPINE) { //then its not good enough to just send to a spine with a connection to the group, it has to go to a spine that has a direct conn to the spec router id - // for(int i = 0; i < possible_next_conns_to_group.size(); i++) - // { - // Connection poss_next_spine = possible_next_conns_to_group[i]; - // vector< Connection> connecting_conns_to_spec = connManagerList[poss_next_spine.dest_gid].get_connections_to_gid(specific_router_id, CONN_GLOBAL); - // if (connecting_conns_to_spec.size() > 0) { //then poss_next_spine is a valid next stop, add my connections to it as possible next stops - // possible_next_conns.push_back(poss_next_spine); - // } - // } - // } - if (specific_router_type == SPINE) { //for now let's allow our spine routers to just send to the group, let them route it to the spec router - possible_next_conns = possible_next_conns_to_group; + return possible_next_conns_to_group; + } + else if (s->dfp_router_type == SPINE) { + return s->connMan->get_connections_to_group(fdest_group_id); + } + } + else { + assert(my_group_id == fdest_group_id); + if (s->dfp_router_type == SPINE) { + vector< Connection > possible_next_conns = s->connMan->get_connections_to_gid(fdest_router_id, CONN_LOCAL); + return possible_next_conns; + } + else { + assert(s->dfp_router_type == LEAF); + + if (my_router_id != fdest_router_id) { //then we're also the source group and we need to send to any spine in our group + assert(my_group_id == origin_group_id); + return s->connMan->get_connections_by_type(CONN_LOCAL); } - else { //then just sending to the group is good enough - possible_next_conns = possible_next_conns_to_group; + else { //then we're the dest router + assert(my_router_id == fdest_router_id); + vector< Connection > empty; + return empty; } - } + } } - - if (possible_next_conns.size() < 1) - tw_error(TW_LOC,"didn't pick any possible next stops!\n"); - - return possible_next_conns; - + vector< Connection > empty; + return empty; } -//always uses 4 rngs + static Connection do_dfp_prog_adaptive_routing(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, int fdest_router_id) { int my_router_id = s->router_id; @@ -2972,14 +2996,14 @@ static Connection do_dfp_prog_adaptive_routing(router_state *s, tw_bf *bf, termi int adaptive_threshold = s->params->adaptive_threshold; bool next_hop_is_global = false; - if (msg->intm_rtr_id == my_router_id) //then we are the intermediate router and need to forward to fdest on the upward channel (VL 1) - msg->dfp_upward_channel_flag = 1; - //The check for dest group local routing has already been completed at this point Connection nextStopConn; - vector< Connection > poss_min_next_stops = get_possible_stops_to_specific_router(s, bf, msg, lp, fdest_router_id); - vector< Connection > poss_intm_next_stops = get_possible_stops_to_specific_router(s, bf, msg, lp, msg->intm_rtr_id); + vector< Connection > poss_min_next_stops = get_legal_minimal_stops(s, bf, msg, lp, fdest_router_id); + vector< Connection > poss_intm_next_stops = get_legal_nonminimal_stops(s, bf, msg, lp, poss_min_next_stops, fdest_router_id); + + if ( (poss_min_next_stops.size() == 0) && (poss_intm_next_stops.size() == 0)) + tw_error(TW_LOC, "No possible next stops!!!"); Connection best_min_conn, best_intm_conn; ConnectionType conn_type_of_mins, conn_type_of_intms; @@ -3000,7 +3024,6 @@ static Connection do_dfp_prog_adaptive_routing(router_state *s, tw_bf *bf, termi else best_intm_conn = get_absolute_best_connection_from_conns(s, bf, msg, lp, poss_intm_next_stops); - //if the best is intermediate, encode the intermediate router id in the message, set path type to non minimal int min_score = dfp_score_connection(s, bf, msg, lp, best_min_conn, C_MIN); int intm_score = dfp_score_connection(s, bf, msg, lp, best_intm_conn, C_NONMIN); @@ -3010,37 +3033,39 @@ static Connection do_dfp_prog_adaptive_routing(router_state *s, tw_bf *bf, termi route_to_fdest = true; } else { - if (msg->path_type == MINIMAL) { //we need to evaluate whether to stay on the minimal path or deviate - if (scoring_preference == LOWER) { - if (min_score <= adaptive_threshold) { - // printf("CHOOSE MIN: Min Score=%d\tNonmin Score=%d (Adaptive Threshold =%d)\n",min_score, intm_score,adaptive_threshold); - route_to_fdest = true; - } - else if (min_score <= intm_score) { - // printf("CHOOSE MIN: Min Score=%d\tNonmin Score=%d\n",min_score, intm_score); - route_to_fdest = true; - } - else { //Changing to a nonminimal path! - // printf("CHOOSE NONMIN: Min Score=%d\tNonmin Score=%d\n",min_score, intm_score); - msg->path_type = NON_MINIMAL; - - } + if (scoring_preference == LOWER) { + if (min_score <= adaptive_threshold) { + route_to_fdest = true; } - else { //HIGHER is better - if (adaptive_threshold > 0) - tw_error(TW_LOC, "Adaptive threshold not compatible with HIGHER score preference yet\n"); //TODO fix this - if (min_score >= intm_score) { - route_to_fdest = true; - } - else { //changing to a nonminimal path! - msg->path_type = NON_MINIMAL; - } + else if (min_score <= intm_score) { + route_to_fdest = true; + } + else { //Changing to a nonminimal path! + msg->path_type = NON_MINIMAL; + } + } + else { //HIGHER is better + if (adaptive_threshold > 0) + tw_error(TW_LOC, "Adaptive threshold not compatible with HIGHER score preference yet\n"); //TODO fix this + if (min_score >= intm_score) { + route_to_fdest = true; + } + else { //changing to a nonminimal path! + msg->path_type = NON_MINIMAL; } } - //we don't need to do anything if the path type is already non-minimal. "Don't revisit decision to route non-minimally after it's been made to be non-minimal" } + if (route_to_fdest && (poss_min_next_stops.size() == 0)) + route_to_fdest = false; + if (route_to_fdest && (poss_intm_next_stops.size() == 0)) + route_to_fdest = true; + + if (route_to_fdest){ + if (in_intermediate_group == true) + msg->dfp_upward_channel_flag = 1; //if we're in an intermediate group and take a turn toward the dest group, we flip the flag! + nextStopConn = best_min_conn; } else { @@ -3063,15 +3088,15 @@ static Connection do_dfp_routing(router_state *s, int origin_group_id = msg->origin_router_id / s->params->num_routers; bool in_intermediate_group = (my_group_id != origin_group_id) && (my_group_id != fdest_group_id); - - if (s->router_id == msg->origin_router_id) { //then we havent picked an intermediate router yet - msg->intm_rtr_id = dfp_pick_intermediate_router(s, bf, msg, lp, origin_group_id, fdest_group_id); - } - if ( (my_group_id == origin_group_id) && (s->dfp_router_type == SPINE) ) { - if (s->params->source_group_rerouting == 0 && isRoutingAdaptive(routing)) { //if we cannot reroute within our source group, we have to pick a new intermediate router - dfp_reselect_intermediate_router(s, bf, msg, lp, origin_group_id, fdest_group_id); //alters msg->intm_rtr_id to be more appropriate - } - } + + // if (s->router_id == msg->origin_router_id) { //then we havent picked an intermediate router yet + // msg->intm_rtr_id = dfp_pick_intermediate_router(s, bf, msg, lp, origin_group_id, fdest_group_id); + // } + // if ( (my_group_id == origin_group_id) && (s->dfp_router_type == SPINE) ) { + // if (s->params->source_group_rerouting == 0 && isRoutingAdaptive(routing)) { //if we cannot reroute within our source group, we have to pick a new intermediate router + // dfp_reselect_intermediate_router(s, bf, msg, lp, origin_group_id, fdest_group_id); //alters msg->intm_rtr_id to be more appropriate + // } + // } Connection nextStopConn; //the connection that we will forward the packet to @@ -3084,7 +3109,7 @@ static Connection do_dfp_routing(router_state *s, return best_min_conn; } else if (my_group_id == fdest_group_id) { //then we just route minimally - vector< Connection > poss_next_stops = get_possible_stops_to_specific_router(s, bf, msg, lp, fdest_router_id); + vector< Connection > poss_next_stops = get_legal_minimal_stops(s, bf, msg, lp, fdest_router_id); if (poss_next_stops.size() < 1) tw_error(TW_LOC, "DEAD END WHEN ROUTING LOCALLY\n"); @@ -3102,7 +3127,7 @@ static Connection do_dfp_routing(router_state *s, } else if (isRoutingMinimal(routing)) { - vector< Connection > poss_next_stops = get_possible_stops_to_specific_router(s, bf, msg, lp, fdest_router_id); + vector< Connection > poss_next_stops = get_legal_minimal_stops(s, bf, msg, lp, fdest_router_id); if (poss_next_stops.size() < 1) tw_error(TW_LOC, "MINIMAL DEAD END\n"); @@ -3116,47 +3141,35 @@ static Connection do_dfp_routing(router_state *s, } else { //routing algorithm is specified in routing - assert( (routing == NON_MINIMAL_LEAF) || (routing == NON_MINIMAL_SPINE) ); bool route_to_fdest = false; - - if(s->dfp_router_type == LEAF) { - if (s->router_id == msg->intm_rtr_id) { //then we are the intermediate router - msg->dfp_upward_channel_flag = 1; //from here on the packet should be routed to fdest_router_id + if (routing == NON_MINIMAL_LEAF) { + if (msg->dfp_upward_channel_flag == 1) //we only route to minimal if this flag is true, otherwise we keep routing nonminimally route_to_fdest = true; - } - else if (msg->dfp_upward_channel_flag == 1) { //then we have already visited the intermediate router - route_to_fdest = true; - } - else { //then we still need to go to the intermediate router - //do nothing for now - if (msg->intm_rtr_id == -1) - tw_error(TW_LOC, "intm router id not set!"); + + else if (s->dfp_router_type == LEAF) { + if (in_intermediate_group) { //then we are the intermediate router + msg->dfp_upward_channel_flag = 1; + route_to_fdest = true; + } } } - else { - assert(s->dfp_router_type == SPINE); - if (s->router_id == msg->intm_rtr_id) { //then we are the intermediate router - msg->dfp_upward_channel_flag = 1; - route_to_fdest = true; - } - else if (msg->dfp_upward_channel_flag == 1) { //then we have already visited the intermediate router - route_to_fdest = true; - } - else { //then we still need to go to the intermediate router - if (msg->intm_rtr_id == -1) - tw_error(TW_LOC, "intm router id not set!"); - } + else if (routing == NON_MINIMAL_SPINE) { + tw_error(TW_LOC, "nonminimal spine routing not yet supported"); } + vector< Connection > poss_min_next_stops = get_legal_minimal_stops(s, bf, msg, lp, fdest_router_id); + vector< Connection > poss_intm_next_stops = get_legal_nonminimal_stops(s, bf, msg, lp, poss_min_next_stops, fdest_router_id); + + vector< Connection > poss_next_stops; if (route_to_fdest) { - poss_next_stops = get_possible_stops_to_specific_router(s, bf, msg, lp, fdest_router_id); + poss_next_stops = poss_min_next_stops; if (poss_next_stops.size() < 1) tw_error(TW_LOC, "Dead end when finding stops to fdest router"); } else { //then we need to be going toward the intermediate router msg->path_type = NON_MINIMAL; - poss_next_stops = get_possible_stops_to_specific_router(s, bf, msg, lp, msg->intm_rtr_id); + poss_next_stops = poss_intm_next_stops; if (poss_next_stops.size() < 1) tw_error(TW_LOC, "Dead end when finding stops to intermediate router"); } @@ -3329,7 +3342,7 @@ static void router_packet_receive(router_state *s, tw_bf *bf, terminal_plus_mess Connection next_stop_conn = do_dfp_routing(s, bf, &(cur_chunk->msg), lp, dest_router_id); if (s->connMan->is_any_connection_to(next_stop_conn.dest_gid) == false) - tw_error(TW_LOC, "Router does not have a connection to chosen destination\n"); + tw_error(TW_LOC, "Router %d does not have a connection to chosen destination %d\n", s->router_id, next_stop_conn.dest_gid); output_port = next_stop_conn.port;