diff --git a/codes/connection-manager.h b/codes/connection-manager.h new file mode 100644 index 0000000000000000000000000000000000000000..3be6129fcb020b8268982e178876e8c4f8bd26fd --- /dev/null +++ b/codes/connection-manager.h @@ -0,0 +1,487 @@ +#ifndef CONNECTION_MANAGER_H +#define CONNECTION_MANAGER_H + +/** + * connection-manager.h -- Simple, Readable, Connection management interface + * Neil McGlohon + * + * Copyright (c) 2018 Rensselaer Polytechnic Institute + */ +#include +#include +#include "codes/codes.h" +#include "codes/model-net.h" + + +using namespace std; + +/** + * @brief Enum differentiating local router connection types from global. + * Local connections will have router IDs ranging from [0,num_router_per_group) + * whereas global connections will have router IDs ranging from [0,total_routers) + */ +enum ConnectionType +{ + CONN_LOCAL = 1, + CONN_GLOBAL, + CONN_TERMINAL +}; + +/** + * @brief Struct for connection information. + */ +struct Connection +{ + int port; //port ID of the connection + int src_lid; //local id of the source + int src_gid; //global id of the source + int src_group_id; //group id of the source + int dest_lid; //local id of the destination + int dest_gid; //global id of the destination + int dest_group_id; //group id of the destination + ConnectionType conn_type; //type of the connection: CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL +}; + +/** + * @class ConnectionManager + * + * @brief + * This class is meant to make organization of the connections between routers more + * streamlined. It provides a simple, readable interface which helps reduce + * semantic errors during development. + * + * @note + * This class was designed with dragonfly type topologies in mind. Certain parts may not + * make sense for other types of topologies, they might work fine, but no guarantees. + * + * @note + * There is the property intermediateRouterToGroupMap and related methods that are implemented but the + * logistics to get this information from input file is more complicated than its worth so I have commented + * them out. + * + * @note + * This class assumes that each router group has the same number of routers in it: _num_routers_per_group. + */ +class ConnectionManager { + map< int, vector< Connection > > intraGroupConnections; //direct connections within a group - IDs are group local - maps local id to list of connections to it + map< int, vector< Connection > > globalConnections; //direct connections between routers not in same group - IDs are global router IDs - maps global id to list of connections to it + map< int, vector< Connection > > terminalConnections; //direct connections between this router and its compute node terminals - maps terminal id to connections to it + + map< int, Connection > _portMap; //Mapper for ports to connections + + // map< int, vector< Connection > > intermediateRouterToGroupMap; //maps group id to list of routers that connect to it. + // //ex: intermediateRouterToGroupMap[3] returns a vector + // //of connections from this router to routers that have + // //direct connections to group 3 + + int _source_id_local; //local id (within group) of owner of this connection manager + int _source_id_global; //global id (not lp gid) of owner of this connection manager + int _source_group; //group id of the owner of this connection manager + + int _used_intra_ports; //number of used ports for intra connections + int _used_inter_ports; //number of used ports for inter connections + int _used_terminal_ports; //number of used ports for terminal connections + + int _max_intra_ports; //maximum number of ports for intra connecitons + int _max_inter_ports; //maximum number of ports for inter connections + int _max_terminal_ports; //maximum number of ports for terminal connections. + + int _num_routers_per_group; //number of routers per group - used for turning global ID into local and back + +public: + ConnectionManager(int src_id_local, int src_id_global, int src_group, int max_intra, int max_inter, int max_term, int num_router_per_group); + + /** + * @brief Adds a connection to the manager + * @param dest_gid the global ID of the destination router + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + */ + void add_connection(int dest_gid, ConnectionType type); + + // /** + // * @brief adds knowledge of what next hop routers have connections to specific groups + // * @param local_intm_id the local intra group id of the router that has the connection to dest_group_id + // * @param dest_group_id the id of the group that the connection goes to + // */ + // void add_route_to_group(int local_intm_id, int dest_group_id); + + // /** + // * @brief returns a vector of connections to routers that have direct connections to the specified group id + // * @param dest_group_id the id of the destination group that all connections returned have a direct connection to + // */ + // vector< Connection > get_intm_conns_to_group(int dest_group_id); + + // /** + // * @brief returns a vector of local router ids that have direct connections to the specified group id + // * @param dest_group_id the id of the destination group that all routers returned have a direct connection to + // * @note if a router has multiple intra group connections to a single router and that router has a connection + // * to the dest group then that router will appear multiple times in the returned vector. + // */ + // vector< int > get_intm_routers_to_group(int dest_group_id) + + /** + * @brief get the source ID of the owner of the manager + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + */ + int get_source_id(ConnectionType type); + + /** + * @brief get the port(s) associated with a specific destination ID + * @param dest_id the ID (local or global depending on type) of the destination + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + */ + vector get_ports(int dest_id, ConnectionType type); + + /** + * @brief get the connection associated with a specific port number + * @param port the enumeration of the port in question + */ + Connection get_connection_on_port(int port); + + /** + * @brief returns true if a connection exists in the manager from the source to the specified destination ID BY TYPE + * @param dest_id the ID of the destination depending on the type + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + * @note Will not return true if dest_id is within own group and type is CONN_GLOBAL, see is_any_connection_to() + */ + bool is_connected_to_by_type(int dest_id, ConnectionType type); + + /** + * @brief returns true if any connection exists in the manager from the soruce to the specified global destination ID + * @param dest_global_id the global id of the destination + * @note This is meant to allow for a developer to determine connectivity just from the global ID, even if the two entities + * are connected by a local or terminal connection. + */ + bool is_any_connection_to(int dest_global_id); + + /** + * @brief returns the total number of used ports by the owner of the manager + */ + int get_total_used_ports(); + + /** + * @brief returns the number of used ports for a specific connection type + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + */ + int get_used_ports_for(ConnectionType type); + + /** + * @brief returns the type of connection associated with said port + * @param port_num the number of the port in question + */ + ConnectionType get_port_type(int port_num); + + /** + * @brief returns a vector of connections to the destination ID based on the connection type + * @param dest_id the ID of the destination depending on the type + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + */ + vector< Connection > get_connections_to_gid(int dest_id, ConnectionType type); + + /** + * @brief returns a vector of connections to the destination group. connections will be of type CONN_GLOBAL + * @param dest_group_id the id of the destination group + */ + vector< Connection > get_connections_to_group(int dest_group_id); + + /** + * @brief returns a vector of all connections to routers via type specified. + * @param type the type of the connection, CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL + * @note this will return connections to same destination on different ports as individual connections + */ + vector< Connection > get_connections_by_type(ConnectionType type); + + /** + * @brief prints out the state of the connection manager + */ + void print_connections(); +}; + + +//******************* BEGIN IMPLEMENTATION ******************************************************** + +//******************* Connection Manager Implementation ******************************************* +ConnectionManager::ConnectionManager(int src_id_local, int src_id_global, int src_group, int max_intra, int max_inter, int max_term, int num_router_per_group) +{ + _source_id_local = src_id_local; + _source_id_global = src_id_global; + _source_group = src_group; + + _used_intra_ports = 0; + _used_inter_ports = 0; + _used_terminal_ports = 0; + + _max_intra_ports = max_intra; + _max_inter_ports = max_inter; + _max_terminal_ports = max_term; + + _num_routers_per_group = num_router_per_group; +} + +void ConnectionManager::add_connection(int dest_gid, ConnectionType type) +{ + Connection conn; + conn.src_lid = _source_id_local; + conn.src_gid = _source_id_global; + conn.src_group_id = _source_group; + conn.conn_type = type; + conn.dest_lid = dest_gid % _num_routers_per_group; + conn.dest_gid = dest_gid; + conn.dest_group_id = dest_gid / _num_routers_per_group; + + switch (type) + { + case CONN_LOCAL: + conn.port = this->get_used_ports_for(CONN_LOCAL); + intraGroupConnections[conn.dest_lid].push_back(conn); + _used_intra_ports++; + break; + + case CONN_GLOBAL: + conn.port = _max_intra_ports + this->get_used_ports_for(CONN_GLOBAL); + globalConnections[conn.dest_gid].push_back(conn); + _used_inter_ports++; + break; + + case CONN_TERMINAL: + conn.port = _max_intra_ports + _max_inter_ports + this->get_used_ports_for(CONN_TERMINAL); + conn.dest_group_id = _source_group; + terminalConnections[conn.dest_gid].push_back(conn); + _used_terminal_ports++; + break; + + default: + assert(false); + // TW_ERROR(TW_LOC, "add_connection(dest_id, type): Undefined connection type\n"); + } + + _portMap[conn.port] = conn; +} + +// void ConnectionManager::add_route_to_group(Connection conn, int dest_group_id) +// { +// intermediateRouterToGroupMap[dest_group_id].push_back(conn); +// } + +// vector< Connection > ConnectionManager::get_intm_conns_to_group(int dest_group_id) +// { +// return intermediateRouterToGroupMap[dest_group_id]; +// } + +// vector< int > ConnectionManager::get_intm_routers_to_group(int dest_group_id) +// { +// vector< Connection > intm_router_conns = get_intm_conns_to_group(dest_group_id); + +// vector< int > loc_intm_router_ids; +// vector< Connection >::iterator it; +// for(it = intm_router_conns.begin(); it != intm_router_conns.end(); it++) +// { +// loc_intm_router_ids.push_back((*it).other_id); +// } +// return loc_intm_router_ids; +// } + +int ConnectionManager::get_source_id(ConnectionType type) +{ + switch (type) + { + case CONN_LOCAL: + return _source_id_local; + case CONN_GLOBAL: + return _source_id_global; + default: + assert(false); + // TW_ERROR(TW_LOC, "get_source_id(type): Unsupported connection type\n"); + } +} + +vector ConnectionManager::get_ports(int dest_id, ConnectionType type) +{ + vector< Connection > conns = this->get_connections_to_gid(dest_id, type); + + vector< int > ports_used; + vector< Connection >::iterator it = conns.begin(); + for(; it != conns.end(); it++) { + ports_used.push_back((*it).port); //add port from connection list to the used ports list + } + return ports_used; +} + +Connection ConnectionManager::get_connection_on_port(int port) +{ + return _portMap[port]; +} + +bool ConnectionManager::is_connected_to_by_type(int dest_id, ConnectionType type) +{ + switch (type) + { + case CONN_LOCAL: + if (intraGroupConnections.find(dest_id) != intraGroupConnections.end()) + return true; + break; + case CONN_GLOBAL: + if (globalConnections.find(dest_id) != globalConnections.end()) + return true; + break; + case CONN_TERMINAL: + if (terminalConnections.find(dest_id) != terminalConnections.end()) + return true; + break; + default: + assert(false); + // TW_ERROR(TW_LOC, "get_used_ports_for(type): Undefined connection type\n"); + } + return false; +} + +bool ConnectionManager::is_any_connection_to(int dest_global_id) +{ + int local_id = dest_global_id % _num_routers_per_group; + if (intraGroupConnections.find(local_id) != intraGroupConnections.end()) + return true; + if (globalConnections.find(dest_global_id) != globalConnections.end()) + return true; + if (terminalConnections.find(dest_global_id) != terminalConnections.end()) + return true; + + return false; +} + +int ConnectionManager::get_total_used_ports() +{ + return _used_intra_ports + _used_inter_ports + _used_terminal_ports; +} + +int ConnectionManager::get_used_ports_for(ConnectionType type) +{ + switch (type) + { + case CONN_LOCAL: + return _used_intra_ports; + case CONN_GLOBAL: + return _used_inter_ports; + case CONN_TERMINAL: + return _used_terminal_ports; + default: + assert(false); + // TW_ERROR(TW_LOC, "get_used_ports_for(type): Undefined connection type\n"); + } +} + +ConnectionType ConnectionManager::get_port_type(int port_num) +{ + return _portMap[port_num].conn_type; +} + + +vector< Connection > ConnectionManager::get_connections_to_gid(int dest_gid, ConnectionType type) +{ + switch (type) + { + case CONN_LOCAL: + return intraGroupConnections[dest_gid%_num_routers_per_group]; + case CONN_GLOBAL: + return globalConnections[dest_gid]; + case CONN_TERMINAL: + return terminalConnections[dest_gid]; + default: + assert(false); + // TW_ERROR(TW_LOC, "get_connections(type): Undefined connection type\n"); + } +} + +vector< Connection > ConnectionManager::get_connections_to_group(int dest_group_id) +{ + vector< Connection > conns_to_group; + + map< int, vector< Connection > >::iterator it = globalConnections.begin(); + for(; it != globalConnections.end(); it++) //iterate over each router that is connected to source + { + vector< Connection >::iterator conns_to_router; + for(conns_to_router = (it->second).begin(); conns_to_router != (it->second).end(); conns_to_router++) //iterate over each connection to a specific router + { + if ((*conns_to_router).dest_group_id == dest_group_id) { + conns_to_group.push_back(*conns_to_router); + } + } + } + return conns_to_group; +} + +vector< Connection > ConnectionManager::get_connections_by_type(ConnectionType type) +{ + map< int, vector< Connection > > theMap; + switch (type) + { + case CONN_LOCAL: + theMap = intraGroupConnections; + break; + case CONN_GLOBAL: + theMap = globalConnections; + break; + case CONN_TERMINAL: + theMap = terminalConnections; + break; + } + + vector< Connection > retVec; + map< int, vector< Connection > >::iterator it; + for(it = theMap.begin(); it != theMap.end(); it++) + { + retVec.insert(retVec.end(), (*it).second.begin(), (*it).second.end()); + } + + return retVec; +} + + +void ConnectionManager::print_connections() +{ + printf("Connections for Router: %d ---------------------------------------\n",_source_id_global); + + int ports_printed = 0; + map::iterator it = _portMap.begin(); + for(; it != _portMap.end(); it++) + { + if ( (ports_printed == 0) && (_used_intra_ports > 0) ) + { + printf(" -- Intra-Group Connections -- \n"); + printf(" Port | Dest_ID | Group\n"); + } + if ( (ports_printed == _used_intra_ports) && (_used_inter_ports > 0) ) + { + printf(" -- Inter-Group Connections -- \n"); + printf(" Port | Dest_ID | Group\n"); + } + if ( (ports_printed == _used_intra_ports + _used_inter_ports) && (_used_terminal_ports > 0) ) + { + printf(" -- Terminal Connections -- \n"); + printf(" Port | Dest_ID | Group\n"); + } + + int port_num = it->first; + int group_id = it->second.dest_group_id; + + int id,gid; + if( get_port_type(port_num) == CONN_LOCAL ) + { + id = it->second.dest_lid; + gid = it->second.dest_gid; + printf(" %d -> (%d,%d) : %d \n", port_num, id, gid, group_id); + + } + + else { + id = it->second.dest_gid; + printf(" %d -> %d : %d \n", port_num, id, group_id); + + } + + ports_printed++; + } +} + + + +#endif /* end of include guard:*/ \ No newline at end of file diff --git a/codes/model-net-lp.h b/codes/model-net-lp.h index e8dd9ef023038b9ad68d045c22e9944a3cc08bb6..d3ca915752e595f65a466650427db925bd2916e0 100644 --- a/codes/model-net-lp.h +++ b/codes/model-net-lp.h @@ -6,11 +6,11 @@ /* This is the base model-net LP that all events pass through before * performing any topology-specific work. Packet scheduling, dealing with - * packet loss (potentially), etc. happens here. - * Additionally includes wrapper event "send" function that all + * packet loss (potentially), etc. happens here. + * Additionally includes wrapper event "send" function that all * events for underlying models must go through */ -#ifndef MODEL_NET_LP_H +#ifndef MODEL_NET_LP_H #define MODEL_NET_LP_H #ifdef __cplusplus @@ -23,6 +23,7 @@ extern "C" { #include "model-net-sched.h" #include "net/dragonfly.h" #include "net/dragonfly-custom.h" +#include "net/dragonfly-plus.h" #include "net/slimfly.h" #include "net/fattree.h" #include "net/loggp.h" @@ -43,10 +44,10 @@ void model_net_base_configure(); /// model developers // Construct a model-net-specific event, analagous to a tw_event_new and -// codes_event_new. The difference here is that we return pointers to +// codes_event_new. The difference here is that we return pointers to // both the message data (to be cast into the appropriate type) and the // pointer to the end of the event struct. -// +// // This function is expected to be called within each specific model-net // method - strange and disturbing things will happen otherwise tw_event * model_net_method_event_new( @@ -131,6 +132,7 @@ typedef struct model_net_wrap_msg { model_net_base_msg m_base; // base lp terminal_message m_dfly; // dragonfly terminal_custom_message m_custom_dfly; // dragonfly-custom + terminal_plus_message m_dfly_plus; // dragonfly plus slim_terminal_message m_slim; // slimfly fattree_message m_fat; // fattree loggp_message m_loggp; // loggp diff --git a/codes/model-net.h b/codes/model-net.h index ee0e1d43d155b6884f56209832f23fcfcca2224f..a6b13f27c1128f105c46c39ddf6c33e84b79ee88 100644 --- a/codes/model-net.h +++ b/codes/model-net.h @@ -41,7 +41,7 @@ extern "C" { /* HACK: there is currently no scheduling fidelity across multiple * model_net_event calls. Hence, problems arise when some LP sends multiple * messages as part of an event and expects FCFS ordering. A proper fix which - * involves model-net LP-level scheduling of requests is ideal, but not + * involves model-net LP-level scheduling of requests is ideal, but not * feasible for now (would basically have to redesign model-net), so expose * explicit start-sequence and stop-sequence markers as a workaround */ @@ -74,6 +74,8 @@ typedef struct mn_stats mn_stats; X(LOGGP, "modelnet_loggp", "loggp", &loggp_method)\ X(EXPRESS_MESH, "modelnet_express_mesh", "express_mesh", &express_mesh_method)\ X(EXPRESS_MESH_ROUTER, "modelnet_express_mesh_router", "express_mesh_router", &express_mesh_router_method)\ + X(DRAGONFLY_PLUS, "modelnet_dragonfly_plus", "dragonfly_plus", &dragonfly_plus_method)\ + X(DRAGONFLY_PLUS_ROUTER, "modelnet_dragonfly_plus_router", "dragonfly_plus_router", &dragonfly_plus_router_method)\ X(MAX_NETS, NULL, NULL, NULL) #define X(a,b,c,d) a, @@ -144,7 +146,7 @@ struct mn_stats long max_event_size; }; -/* Registers all model-net LPs in ROSS. Should be called after +/* Registers all model-net LPs in ROSS. Should be called after * configuration_load, but before codes_mapping_setup */ void model_net_register(); @@ -166,8 +168,8 @@ void model_net_enable_sampling(tw_stime interval, tw_stime end); int model_net_sampling_enabled(void); /* Initialize/configure the network(s) based on the CODES configuration. - * returns an array of the network ids, indexed in the order given by the - * modelnet_order configuration parameter + * returns an array of the network ids, indexed in the order given by the + * modelnet_order configuration parameter * OUTPUT id_count - the output number of networks */ int* model_net_set_params(int *id_count); @@ -189,7 +191,7 @@ void model_net_event_collective_rc( int message_size, tw_lp *sender); -/* allocate and transmit a new event that will pass through model_net to +/* allocate and transmit a new event that will pass through model_net to * arrive at its destination: * * - net_id: the type of network to send this message through. The set of @@ -231,9 +233,9 @@ void model_net_event_collective_rc( // first argument becomes the network ID model_net_event_return model_net_event( int net_id, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, tw_stime offset, int remote_event_size, void const * remote_event, @@ -251,9 +253,9 @@ model_net_event_return model_net_event( model_net_event_return model_net_event_annotated( int net_id, char const * annotation, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, tw_stime offset, int remote_event_size, void const * remote_event, @@ -270,9 +272,9 @@ model_net_event_return model_net_event_mctx( int net_id, struct codes_mctx const * send_map_ctx, struct codes_mctx const * recv_map_ctx, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, tw_stime offset, int remote_event_size, void const * remote_event, @@ -309,7 +311,7 @@ int model_net_get_msg_sz(int net_id); * identical to the sender argument to tw_event_new(). */ /* NOTE: we may end up needing additoinal arguments here to track state for - * reverse computation; add as needed + * reverse computation; add as needed */ DEPRECATED void model_net_event_rc( @@ -333,7 +335,7 @@ void model_net_event_rc2( * Parameters are largely the same as model_net_event, with the following * exceptions: * - final_dest_lp is the lp to pull data from - * - self_event_size, self_event are applied at the requester upon receipt of + * - self_event_size, self_event are applied at the requester upon receipt of * the payload from the dest */ model_net_event_return model_net_pull_event( @@ -383,7 +385,7 @@ void model_net_pull_event_rc( * model-net implementation (currently implemented as a set of translation-unit * globals). Upon a subsequent model_net_*event* call, the context is consumed * and reset to an unused state. - * + * * NOTE: this call MUST be placed in the same calling context as the subsequent * model_net_*event* call. Otherwise, the parameters are not guaranteed to work * on the intended event, and may possibly be consumed by another, unrelated diff --git a/codes/net/dragonfly-plus.h b/codes/net/dragonfly-plus.h new file mode 100644 index 0000000000000000000000000000000000000000..602dbc710bc534c0347db8ff878216ed5e9dde94 --- /dev/null +++ b/codes/net/dragonfly-plus.h @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2014 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef DRAGONFLY_PLUS_H +#define DRAGONFLY_PLUS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct terminal_plus_message terminal_plus_message; + +/* this message is used for both dragonfly compute nodes and routers */ +struct terminal_plus_message +{ + /* magic number */ + int magic; + /* flit travel start time*/ + tw_stime travel_start_time; + /* packet ID of the flit */ + unsigned long long packet_ID; + /* event type of the flit */ + short type; + /* category: comes from codes */ + char category[CATEGORY_NAME_MAX]; + /* store category hash in the event */ + uint32_t category_hash; + /* final destination LP ID, this comes from codes can be a server or any other LP type*/ + tw_lpid final_dest_gid; + /*sending LP ID from CODES, can be a server or any other LP type */ + tw_lpid sender_lp; + tw_lpid sender_mn_lp; // source modelnet id + /* destination terminal ID of the dragonfly */ + tw_lpid dest_terminal_id; + /* source terminal ID of the dragonfly */ + unsigned int src_terminal_id; + /* message originating router id. MM: Can we calculate it through + * sender_mn_lp??*/ + unsigned int origin_router_id; + + /* number of hops traversed by the packet */ + short my_N_hop; + short my_l_hop, my_g_hop; + short saved_channel; + short saved_vc; + + int next_stop; + + short nonmin_done; + /* Intermediate LP ID from which this message is coming */ + unsigned int intm_lp_id; + /* last hop of the message, can be a terminal, local router or global router */ + short last_hop; + /* For routing */ + int saved_src_dest; + int saved_src_chan; + + //DFP Specific Routing + int intm_rtr_id; //Router ID of the intermediate router for nonminimal routes + int intm_group_id; //Group ID of the intermediate router for nonminimal routes + + short dfp_upward_channel_flag; + + int dfp_dest_terminal_id; //this is the terminal id in the dfp network in range [0-total_num_terminals) + + uint32_t chunk_id; + uint32_t packet_size; + uint32_t message_id; + uint32_t total_size; + + int remote_event_size_bytes; + int local_event_size_bytes; + + // For buffer message + short vc_index; + int output_chan; + model_net_event_return event_rc; + int is_pull; + uint32_t pull_size; + + /* for reverse computation */ + int path_type; + tw_stime saved_available_time; + tw_stime saved_avg_time; + tw_stime saved_rcv_time; + tw_stime saved_busy_time; + tw_stime saved_total_time; + tw_stime saved_sample_time; + tw_stime msg_start_time; +}; + +#ifdef __cplusplus +} +#endif + +#endif /* end of include guard: DRAGONFLY_H */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ft=c ts=8 sts=4 sw=4 expandtab + */ diff --git a/scripts/dragonfly-plus/dragonfly-plus-topo-gen-v2.py b/scripts/dragonfly-plus/dragonfly-plus-topo-gen-v2.py new file mode 100644 index 0000000000000000000000000000000000000000..b2860ab3577172314a1745be4cacd9ba1752f34a --- /dev/null +++ b/scripts/dragonfly-plus/dragonfly-plus-topo-gen-v2.py @@ -0,0 +1,609 @@ +# Copyright 2018 - Neil McGlohon +# mcglon@rpi.edu +# An Object Oriented approach to topology generation - Allows for a logical generation that may be more complicated than v1 which used assumptions +# In hindsight this was a lot more complicated than I intended. It was looking to solve a complex problem that turned out to be invalid from the beginning. + +### USAGE ### +# Correct usage: python3 script.py +### ### + +import sys +from enum import Enum +import struct +import numpy as np +argv = sys.argv + +class Loudness(Enum): + DEBUG = 0 #prints all output + EXTRA_LOUD = 1 #prints every single connection too + LOUD = 2 #prints group connections and other similarly verbose things + STANDARD = 3 #prints what step its working on + QUIET = 4 #print no output + +global DRYRUN +global LOUDNESS +global SHOW_ADJACENCY +global NO_OUTPUT_FILE + +LOUDNESS = Loudness.STANDARD +DRYRUN = 0 +SHOW_ADJACENCY = 0 +NO_OUTPUT_FILE = 0 + +global desired_num_gc_bg + + +if sys.version_info[0] < 3: + raise Exception("Python 3 or a more recent version is required.") + +class ConnType(Enum): + LOCAL = 0 + GLOBAL = 1 + +class RouterType(Enum): + LEAF = 0 + SPINE = 1 + +class AdjacencyType(Enum): + LOCAL_LOCAL = 0 # size = (num_routers_pg, num_routers_pg) + LOCAL_ONLY_GID = 1 # size = (total_routers, total_routers) but only the submatrix for the group is filled in. i.e. local connections only by gid + GLOBAL_ONLY = 2 # size = (total_routers, total_routers) but only the global connecitons are included + ALL_CONNS = 3 # size = (total_routers, total_routers) with global connections and local connections. Actual FULL adjacency matrix + + +def log(mystr, loudness): + global LOUDNESS + if loudness.value >= LOUDNESS.value: + print(mystr) + +def sortBySource(conn): #returns the key by which connections should be sorted (source id, local or global) + if conn.connType is ConnType.LOCAL: + return conn.src_router.local_id + else: + return conn.src_router.gid + +def sortByDest(conn): + if conn.connType is ConnType.LOCAL: + return conn.dest_router.local_id + else: + return conn.dest_router.gid + +def convertLocalIDtoGID(router_lid, router_group_num, routers_per_group): + return router_lid + router_group_num * routers_per_group + +def convertGIDtoLocalID(router_gid, routers_per_group): + return router_gid % routers_per_group + +class DragonflyPlusNetwork(object): + def __init__(self,num_groups=5, num_spine_pg=2, num_leaf_pg=2, router_radix=4, num_hosts_per_leaf=2): + log("New Dragonfly Plus Network", Loudness.STANDARD) + self.num_groups = num_groups + self.num_spine_pg = num_spine_pg + self.num_leaf_pg = num_leaf_pg + self.router_radix = router_radix + + self.num_global_links_per_spine = router_radix - num_leaf_pg + self.num_global_links_pg = self.num_global_links_per_spine * num_spine_pg + + largest_num_groups = (router_radix//2)**2 + 1 + global desired_num_gc_bg + self.desired_num_global_links_between_groups = desired_num_gc_bg + self.num_global_links_between_groups = (largest_num_groups-1)//(num_groups-1) + + self.num_hosts_per_leaf = num_hosts_per_leaf #just for stats, not used in the actual topology export + + self.localConnectionsOutput = [] + self.globalConnectionsOutput = [] + self.groups = [] + + self.precheckForErrors() + if not DRYRUN: + self.initializeGroups() + self.generateLocalGroupConnections() + self.generateGlobalGroupConnections() + self.verifyTopology() + + def getSummary(self): + outStr = "\nDragonfly Plus Network:\n" + outStr += "\tNumber of Groups: %d\n" % self.num_groups + outStr += "\tRouter Radix: %d\n" % self.router_radix + outStr += "\tNumber Spine Per Group: %d\n" % self.num_spine_pg + outStr += "\tNumber Leaf Per Group: %d\n" % self.num_leaf_pg + outStr += "\tNumber Terminal Per Leaf: %d\n" % self.num_hosts_per_leaf + outStr += "\n" + outStr += "\tNumber GC per Spine: %d\n" % self.num_global_links_per_spine + outStr += "\tNumber GC per Group: %d\n" % (self.num_global_links_per_spine * self.num_spine_pg) + outStr += "\tNumber GC between Groups: %d\n" % (self.num_global_links_between_groups) + outStr += "\n" + outStr += "\tTotal Spine: %d\n" % (self.num_spine_pg * self.num_groups) + outStr += "\tTotal Leaf: %d\n" % (self.num_leaf_pg * self.num_groups) + outStr += "\tTotal Routers: %d\n" % ((self.num_leaf_pg + self.num_spine_pg) * self.num_groups) + outStr += "\tTotal Number Terminals: %d\n" % (self.num_leaf_pg * self.num_hosts_per_leaf * self.num_groups) + outStr += "\t" + return outStr + + + def initializeGroups(self): + groups = [] + num_routers_pg = self.num_spine_pg + self.num_leaf_pg + for new_group_id in range(self.num_groups): + newGroup = Group(new_group_id, self.num_global_links_pg, self) + + for local_id in range(num_routers_pg): + router_gid = local_id + (new_group_id * num_routers_pg) + if local_id < self.num_leaf_pg: + intra_radix = self.num_spine_pg + inter_radix = 0 + rt = RouterType.LEAF + + elif local_id >= self.num_leaf_pg: + intra_radix = self.num_leaf_pg + inter_radix = self.getNumGlobalConnsPerSpine() + rt = RouterType.SPINE + + newRouter = Router(router_gid, local_id, new_group_id, intra_radix, inter_radix, rt, self) + newGroup.addRouter(newRouter) + + groups.append(newGroup) + log("Added New Group. ID: %d, Size: %d" % ( new_group_id, len(newGroup.group_routers) ) , Loudness.LOUD) + + self.groups = groups + + def generateLocalGroupConnections(self): + log("Dragonfly Plus Network: Generating Local Group Connections", Loudness.STANDARD) + for group in self.groups: + group.generateLocalConnections() + + def generateGlobalGroupConnections(self): + log("Dragonfly Plus Network: Generating Global Group Connections", Loudness.STANDARD) + for group in self.groups: + other_groups = group.getOtherGroupIDsStartingAfterMe(self.num_groups) + for gl in range(self.num_global_links_pg): + other_group_id = other_groups[gl % len(other_groups)] + + group.addGlobalConnection(self.groups[other_group_id]) + + for group in self.groups: + group.assignGlobalConnectionsToRouters() + + def getNumGlobalConnsPerSpine(self): + return self.num_global_links_pg // self.num_spine_pg + + def getTotalRouters(self): + return self.num_groups * (self.num_spine_pg + self.num_leaf_pg) + + def getTotalNodes(self): + return self.num_hosts_per_leaf * self.num_leaf_pg * self.num_groups + + def getAllSpineRouters(self): + spine_rtrs = [] + for g in self.groups: + for rtr in g.group_routers: + if rtr.routerType is RouterType.SPINE: + spine_rtrs.append(rtr) + return spine_rtrs + + def getAllLeafRouters(self): + leaf_rtrs = [] + for g in self.groups: + for rtr in g.group_routers: + if rtr.routerType is RouterType.LEAF: + leaf_rtrs.append(rtr) + return leaf_rtrs + + def getRouter(self,gid): + router_group_num = gid // (self.num_spine_pg + self.num_leaf_pg) + offset = gid % (self.num_spine_pg + self.num_leaf_pg) + + rtr = self.groups[router_group_num].group_routers[offset] + assert(rtr.gid == gid) + return rtr + + def precheckForErrors(self): + if self.num_global_links_pg % self.desired_num_global_links_between_groups != 0: + raise Exception("DragonflyPlusNetwork: Global Connection Fairness Violation. num_global_links_pg % num_global_links_between_groups != 0") + if self.num_global_links_pg % self.num_spine_pg != 0: #are we fair? + raise Exception("DragonflyPlusNetwork: Global Connection Fairness Violation. num_global_links_pg % num_spine_pg != 0!!!!") + if self.num_leaf_pg > (self.router_radix - self.getNumGlobalConnsPerSpine()): #do we have enough ports left to connect to all the leafs in the group after connecting global conns? + raise Exception("DragonflyPlusNetwork: Inadequate radix for number of global connections per spine router") + + def verifyTopology(self): + A = self.getAdjacencyMatrix(AdjacencyType.ALL_CONNS) + + #verify symmetry. 1 <-> 2 == 2 <-> 1 + log("Verifying Symmetry...", Loudness.STANDARD) + for i in range(A.shape[0]): + for j in range(A.shape[1]): + if A[i][j] != A[j][i]: + raise Exception("DragonflyPlusNetwork: Failed Verification: Topology not symmetric") + + #verify safe radix + log("Verifying Radix Usage...", Loudness.STANDARD) + for row in A: + # log("%d > %d: %s" % (sum(row), self.router_radix, str(sum(row) > self.router_radix) ) ) + if sum(row) > self.router_radix: + raise Exception("DragonflyPlusNetwork: Failed Verification: Router Radix Exceeded") + + #verify fairness + log("Verifying Fairness...", Loudness.STANDARD) + spine_rtrs = self.getAllSpineRouters() + leaf_rtrs = self.getAllLeafRouters() + + #all spine routers have same local and global connections sizes + loc_conns = len(spine_rtrs[0].local_connections) + glob_conns = len(spine_rtrs[0].global_connections) + for s in spine_rtrs: + loc = len(s.local_connections) + glob = len(s.global_connections) + failed = False + if loc is not loc_conns: + failed = True + if glob is not glob_conns: + failed = True + + #all leaf routers have same local and global connections sizes + #all leaf routers have 0 global connections + loc_conns = len(leaf_rtrs[0].local_connections) + glob_conns = 0 + for l in leaf_rtrs: + loc = len(l.local_connections) + glob = len(l.global_connections) + failed = False + if loc is not loc_conns: + failed = True + if glob is not glob_conns: + failed = True + + log("Verifying Dragonfly Nature...", Loudness.STANDARD) + + + if failed: + raise Exception("DragonflyPlusNetwork: Failed Verification: Fairness") + + for g in self.groups: + if len(set(g.groupConns)) != self.num_global_links_pg: + raise Exception("DragonflyPlusNetwork: Not Enough Group Connections") + + log("Verifying Inter Group Connection Uniformity...", Loudness.STANDARD) + num_gc_between_0_1 = len(self.groups[0].getConnectionsToGroup(1)) + for g in self.groups: + other_groups = g.getOtherGroupIDsStartingAfterMe(self.num_groups) + for other_group_id in other_groups: + if len(g.getConnectionsToGroup(other_group_id)) != num_gc_between_0_1: + raise Exception("DragonflyPlusNetwork: Failed Verification: InterGroup Connection Uniformity") + + + + + + + def commitConnection(self,conn, connType): + if connType is ConnType.LOCAL: + self.localConnectionsOutput.append(conn) + else: + self.globalConnectionsOutput.append(conn) + + def writeInterconnectionFile(self, inter_filename): + log("\nWriting out InterConnection File '%s': " % inter_filename, Loudness.STANDARD) + with open(inter_filename, "wb") as fd: + for conn in sorted(self.globalConnectionsOutput,key=sortBySource): + src_gid = conn.src_router.gid + dest_gid = conn.dest_router.gid + fd.write(struct.pack("2i",src_gid,dest_gid)) + + log("INTER %d %d sgrp %d dgrp %d" % (src_gid, dest_gid, conn.src_router.group_id, conn.dest_router.group_id), Loudness.EXTRA_LOUD) + + def writeIntraconnectionFile(self, intra_filename): + log("\nWriting out IntraConnection File '%s': " % intra_filename, Loudness.STANDARD) + with open(intra_filename, "wb") as fd: + for conn in sorted(self.localConnectionsOutput,key=sortBySource): + if conn.src_router.group_id == 0: + src_id = conn.src_router.local_id + dest_id = conn.dest_router.local_id + fd.write(struct.pack("2i",src_id,dest_id)) + log("INTRA %d %d" % (src_id, dest_id), Loudness.EXTRA_LOUD) + + def getAdjacencyMatrix(self, adj_type): + conns = [] + if adj_type is AdjacencyType.LOCAL_LOCAL: + conns.extend(self.localConnectionsOutput) + size = self.num_spine_pg + self.num_leaf_pg + elif adj_type is AdjacencyType.LOCAL_ONLY_GID: + conns.extend(self.localConnectionsOutput) + size = self.num_groups * (self.num_spine_pg + self.num_leaf_pg) + elif adj_type is AdjacencyType.GLOBAL_ONLY: + conns.extend(self.globalConnectionsOutput) + size = self.num_groups * (self.num_spine_pg + self.num_leaf_pg) + elif adj_type is AdjacencyType.ALL_CONNS: + conns.extend(self.localConnectionsOutput) + conns.extend(self.globalConnectionsOutput) + size = self.num_groups * (self.num_spine_pg + self.num_leaf_pg) + + log("A = np.zeros((%d,%d))" % (size,size), Loudness.DEBUG) + A = np.zeros((size,size)) + + for conn in conns: + if adj_type is AdjacencyType.LOCAL_LOCAL: + if conn.src_router.group_id is 0: + A[conn.src_router.local_id][conn.dest_router.local_id] += 1 + else: + A[conn.src_router.gid][conn.dest_router.gid] += 1 + + return A + + +class Group(object): + def __init__(self, group_id, group_global_radix, network): + self.group_id = group_id + self.group_routers = [] #list of routers in this group + self.group_global_radix = group_global_radix #number of connections from this group to other groups + self.used_radix = 0 #used number of connections from this group to other groups + self.network = network + + self.groupConns = [] + + + def addRouter(self,router): + self.group_routers.append(router) + self.used_radix += router.inter_radix + + def addGlobalConnection(self, other_group): + log("Group %d -> Group %d" % (self.group_id, other_group.group_id), Loudness.LOUD) + self.groupConns.append(GroupConnection(self,other_group)) + + def getSpineRouters(self): + return [r for r in self.group_routers if r.routerType is RouterType.SPINE] + + def getLeafRouters(self): + return [r for r in self.group_routers if r.routerType is RouterType.LEAF] + + def getOtherGroupIDsStartingAfterMe(self,num_groups): + my_group_id = self.group_id + all_group_ids = [i for i in range(num_groups) if i != my_group_id] + return np.roll(all_group_ids, -1*my_group_id) + + def getConnectionsToGroup(self,other_group_id): + return [conn for conn in self.groupConns if conn.dest_group.group_id == other_group_id] + + def generateLocalConnections(self): + log("Group %d: generating local connections" % self.group_id, Loudness.LOUD) + + spine_routers = [rtr for rtr in self.group_routers if rtr.routerType == RouterType.SPINE] + leaf_routers = [rtr for rtr in self.group_routers if rtr.routerType == RouterType.LEAF] + + for srtr in spine_routers: + for lrtr in leaf_routers: + srtr.connectTo(lrtr, ConnType.LOCAL) + + def assignGlobalConnectionsToRouters(self): + log("Group %d: assigning global connections" % self.group_id, Loudness.LOUD) + num_routers_pg = self.network.num_leaf_pg + self.network.num_spine_pg + + spine_routers = self.getSpineRouters() + + other_groups = self.getOtherGroupIDsStartingAfterMe(self.network.num_groups) + for i,group_conn in enumerate(self.groupConns): + src_router = spine_routers[int(i // self.network.num_global_links_per_spine)] + dest_group = group_conn.dest_group + dest_group_id = group_conn.dest_group.group_id + local_spinal_id = src_router.local_id - self.network.num_leaf_pg #which spine is the source + dest_spinal_id = self.network.num_spine_pg - local_spinal_id - 1 + dest_local_id = dest_group.getSpineRouters()[dest_spinal_id].local_id + dest_global_id = convertLocalIDtoGID(dest_local_id, dest_group_id, num_routers_pg) + dest_rtr = self.network.getRouter(dest_global_id) + src_router.connectToOneWay(dest_rtr, ConnType.GLOBAL) + + +class Router(object): + def __init__(self, self_gid, self_local_id, group_id, intra_radix, inter_radix, routerType, network): + self.gid = self_gid + self.local_id = self_local_id + self.group_id = group_id + self.intra_radix = intra_radix + self.inter_radix = inter_radix + self.routerType = routerType + self.local_connections = [] + self.global_connections = [] + self.network = network + + log("New Router: GID: %d LID: %d Group %d" % (self.gid, self.local_id, self.group_id), Loudness.DEBUG) + + # local_spinal_id = self.local_id - self.network.num_leaf_pg #what is my ID in terms of num spine + # other_groups = self.network.groups[group_id].getOtherGroupIDsStartingAfterMe(self.network.num_groups) + # other_groups_i_connect_to_std = [g for i,g in enumerate(other_groups) if i] + # other_groups_i_connect_to_start_index_std = (local_spinal_id * self.network.num_spine_pg) % len(other_groups) + # other_groups_i_connect_to_start_index_addtl_gc = (other_groups_i_connect_to_start_index_std - (self.network.num_global_links_per_spine - self.network.num_spine_pg)) % len(other_groups) + + + + + def connectTo(self, other_rtr, connType): + if connType is ConnType.GLOBAL: + assert(self.routerType == RouterType.SPINE) + assert(other_rtr.routerType == RouterType.SPINE) + conn = Connection(self,other_rtr,connType) + oconn = Connection(other_rtr, self,connType) + + self.addConnection(conn, connType) + other_rtr.addConnection(oconn, connType) + + def connectToOneWay(self, other_rtr, connType): #connects without connecting backward - for use if you know your loop will double count + if connType is ConnType.GLOBAL: + assert(self.routerType == RouterType.SPINE) + assert(other_rtr.routerType == RouterType.SPINE) + conn = Connection(self, other_rtr, connType) + self.addConnection(conn, connType) + + def addConnection(self,conn, conntype): + if conntype is ConnType.LOCAL: + if len(self.local_connections) >= self.intra_radix: + raise Exception("Router %d: Cannot add connection, exceeds intra_radix"% self.gid) + self.local_connections.append(conn) + log("New: INTRA %d %d" % (conn.src_router.local_id, conn.dest_router.local_id), Loudness.EXTRA_LOUD) + + elif conntype is ConnType.GLOBAL: + if len(self.global_connections) >= self.inter_radix: + raise Exception("Router %d: Cannot add connection, exceeds inter_radix"% self.gid) + self.global_connections.append(conn) + log("New: INTER %d %d | Group: %d -> %d" % (conn.src_router.gid, conn.dest_router.gid, conn.src_router.group_id, conn.dest_router.group_id), Loudness.EXTRA_LOUD) + else: + raise Exception("Invalid Connection Type") + + self.network.commitConnection(conn, conntype) + + +class Connection(object): + def __init__(self, src_router, dest_router, connType, shifted_by=0): + self.src_router = src_router + self.dest_router = dest_router + self.connType = connType + self.shifted_by = shifted_by + + def __getitem__(self, key): + if type(key) is str: + if key == "src": + return self.src_router + if key == "dest": + return self.dest_router + + elif type(key) is int: + if key is 0: + return self.src_router + elif key is 1: + return self.dest_router + else: + raise IndexError("Connection: index out of range") + + else: + raise KeyError("Connection: Invalid __getitem__() key") + +class GroupConnection(object): + def __init__(self, src_group, dest_group): + self.src_group = src_group + self.dest_group = dest_group + +def parseOptionArguments(): + global DRYRUN + global LOUDNESS + global SHOW_ADJACENCY + global NO_OUTPUT_FILE + + if "--debug" in argv: + LOUDNESS = Loudness.DEBUG + + if "--extra-loud" in argv: + LOUDNESS = Loudness.EXTRA_LOUD + + if "--loud" in argv: + LOUDNESS = Loudness.LOUD + + if "--quiet" in argv: + LOUDNESS = Loudness.QUIET + + if "--dry-run" in argv: + DRYRUN = 1 + + if "--no-output" in argv: + NO_OUTPUT_FILE = 1 + + if "--show-adjacency" in argv: + SHOW_ADJACENCY = 1 + + +#mainV3() assumes things about the network: +# num_spine_pg == num_leaf_pg +# term_per_leaf = num_spine_pg +# num_router_pg = router_radix (half up half down) +# total_gc_per_group = router_radix / 2 +def mainV3(): + parseOptionArguments() + + if DRYRUN or NO_OUTPUT_FILE: + if DRYRUN and NO_OUTPUT_FILE: + option = '--dry-run' + elif DRYRUN: + option = '--dry-run' + elif NO_OUTPUT_FILE: + option = '--no-output' + if(len(argv) < 4): + raise Exception("Correct usage: python %s %s" % (sys.argv[0], option)) + + elif(len(argv) < 5): + raise Exception("Correct usage: python %s " % sys.argv[0]) + + router_radix = int(argv[1]) + num_gc_between_groups = int(argv[2]) + + global desired_num_gc_bg + desired_num_gc_bg = num_gc_between_groups + + largest_num_groups = (router_radix//2)**2 + 1 + num_groups = (largest_num_groups - 1) // num_gc_between_groups + 1 + + num_router_pg = int(router_radix) + num_spine_pg = int(num_router_pg / 2) + num_leaf_pg = num_spine_pg + term_per_leaf = num_spine_pg + + if not (DRYRUN or NO_OUTPUT_FILE): + intra_filename = argv[3] + inter_filename = argv[4] + + + try: + dfp_network = DragonflyPlusNetwork(num_groups, num_spine_pg, num_leaf_pg, router_radix, num_hosts_per_leaf=term_per_leaf) + except Exception as err: + print("ERROR: ",err) + exit(1) + + + if not DRYRUN: + if not NO_OUTPUT_FILE: + dfp_network.writeIntraconnectionFile(intra_filename) + dfp_network.writeInterconnectionFile(inter_filename) + + if LOUDNESS is not Loudness.QUIET: + print(dfp_network.getSummary()) + + if SHOW_ADJACENCY == 1: + print("\nPrinting Adjacency Matrix:") + + np.set_printoptions(linewidth=400,threshold=10000,edgeitems=200) + A = dfp_network.getAdjacencyMatrix(AdjacencyType.ALL_CONNS) + print(A.astype(int)) + + +def mainV2(): + if(len(argv) < 8): + raise Exception("Correct usage: python %s " % sys.argv[0]) + + num_groups = int(argv[1]) + num_spine_pg = int(argv[2]) + num_leaf_pg = int(argv[3]) + router_radix = int(argv[4]) + term_per_leaf = int(argv[5]) + intra_filename = argv[6] + inter_filename = argv[7] + + parseOptionArguments() + + dfp_network = DragonflyPlusNetwork(num_groups, num_spine_pg, num_leaf_pg, router_radix, num_hosts_per_leaf=term_per_leaf) + + if not DRYRUN: + dfp_network.writeIntraconnectionFile(intra_filename) + dfp_network.writeInterconnectionFile(inter_filename) + + if LOUDNESS is not Loudness.QUIET: + print("\nNOTE: THIS STILL CAN'T DO THE MED-LARGE TOPOLOGY RIGHT\n") + + print(dfp_network.getSummary()) + + if SHOW_ADJACENCY == 1: + print("\nPrinting Adjacency Matrix:") + + np.set_printoptions(linewidth=400,threshold=10000,edgeitems=200) + A = dfp_network.getAdjacencyMatrix(AdjacencyType.ALL_CONNS) + print(A.astype(int)) + +if __name__ == '__main__': + mainV3() diff --git a/scripts/dragonfly-plus/dragonfly-plus-topo-gen.py b/scripts/dragonfly-plus/dragonfly-plus-topo-gen.py new file mode 100644 index 0000000000000000000000000000000000000000..3fc2d20893f06e49e385e3c0c42f99cd558cc865 --- /dev/null +++ b/scripts/dragonfly-plus/dragonfly-plus-topo-gen.py @@ -0,0 +1,152 @@ +# Copyright 2017 - Neil McGlohon +# mcglon@rpi.edu + +import sys +from enum import Enum +import struct +import numpy as np +argv = sys.argv + +if sys.version_info[0] < 3: + raise Exception("Python 3 or a more recent version is required.") + + + +class TopSize(Enum): + SMALL = 0 + MEDIUM = 1 + LARGE = 2 + +def main(): + if(len(argv) < 8): + raise Exception("Correct usage: python %s " % sys.argv[0]) + + groups = int(argv[1]) + num_spine_routers = int(argv[2]) + num_leaf_routers = int(argv[3]) + topology_size = TopSize(int(argv[4])) + redundant_global_cons_per = int(argv[5]) + intra = open(argv[6], "wb") + inter = open(argv[7], "wb") + + writeIntra(num_spine_routers, num_leaf_routers, intra) + writeInter(num_spine_routers, num_leaf_routers, topology_size, groups, redundant_global_cons_per, inter) + + intra.close() + inter.close() + + +def getRouterGID(localID, groupNumber, routers_per_group): + return(localID + (groupNumber*routers_per_group)) + + +def writeIntra(num_spine_routers,num_leaf_routers,fd): + total_routers = num_spine_routers + num_leaf_routers + + A = np.zeros((total_routers,total_routers)) #intra adjacency matrix in case you want one + + #for each leaf router, connect it to all spine routers in group + for li in range(num_leaf_routers): + for si in range(num_leaf_routers,total_routers): + A[li][si] = 1 + fd.write(struct.pack("2i",li,si)) + print("INTRA %d %d"%(li,si)) + + #for each spine router, connect it to all leaf routers in group + for si in range(num_leaf_routers,total_routers): + for li in range(num_leaf_routers): + A[si][li] = 1 + fd.write(struct.pack("2i",si,li)) + print("INTRA %d %d"%(si,li)) + +def writeInter(num_spine_routers,num_leaf_routers, topology_size, num_groups, redundant_global_cons_per,fd): + total_routers_per_group = num_spine_routers + num_leaf_routers + global_total_routers = total_routers_per_group * num_groups + global_cons_per = redundant_global_cons_per + 1 + + Ag = np.zeros((global_total_routers,global_total_routers)) + + if (topology_size is TopSize.SMALL) or (topology_size is TopSize.MEDIUM): + #Every spine is connected to every other group + + if (topology_size is TopSize.MEDIUM) and (redundant_global_cons_per > 0): + raise Exception("Error: redundant connections incompatible with Medium topology") + + for source_gi in range(num_groups): + for si in range(num_leaf_routers, total_routers_per_group): + source_id = getRouterGID(si,source_gi,total_routers_per_group) + + for dest_gi in range(num_groups): + if source_gi != dest_gi: + dest_id = getRouterGID(si, dest_gi,total_routers_per_group) + for i in range(global_cons_per): + fd.write(struct.pack("2i",source_id,dest_id)) + print("INTER %d %d srcg %d destg %d"%(source_id,dest_id,source_gi,dest_gi)) + + elif topology_size is TopSize.LARGE: + #Each group is connected to every other group via single connection on individual spines + ind_radix = 2 * num_leaf_routers #TODO don't assume that the radix is half down half up + ind_up_radix = int(ind_radix/2) + num_other_groups = num_groups - 1 + + if(num_other_groups%num_spine_routers != 0): + raise Exception("ERROR: Assymetrical - num_other_groups\%num_spine_routers != 0") #TODO Consider allowing such a setting? + + if(num_other_groups != (num_spine_routers**2)): + raise Exception("ERROR: Assymetrical - num_other_groups != num_spine_routers^2") + + if num_other_groups != (num_spine_routers * ind_up_radix): + raise Exception("Error: Invalid topology - num groups exceeds group upward radix") + + if ind_up_radix > num_groups: + raise Exception("ERROR: The number of global connections per spine router exceeds the number of groups. Not Large Topology!") + + if ind_up_radix != num_spine_routers: + raise Exception("ERROR: the upward radix must equal the number of spine routers") + + interlinks = [] + + spine_router_ids = [i for i in range(global_total_routers) if (i % total_routers_per_group) >= num_leaf_routers] + all_groups = [i for i in range(num_groups)] + for i,source_id in enumerate(spine_router_ids): + source_group_id = int(source_id / total_routers_per_group) + spine_local_id = source_id % total_routers_per_group + xth_spine = spine_local_id - num_leaf_routers #if we were only counting spine routers, this means that I am the xth spine where x is this value + + other_groups = [j for j in range(num_groups) if j != source_group_id] #ids of groups not the source group + my_other_groups = [] #the specific groups that this spine router will connect to + for ii in range(ind_up_radix): + index = (source_group_id+1 + ii + xth_spine*ind_up_radix) % num_groups + my_other_groups.append(all_groups[index]) + + dest_spinal_offset = (num_spine_routers-1) - xth_spine #which xth spine will the spine router connect to in the dest group. + for dest_group_id in my_other_groups: + dest_group_offset = dest_group_id * total_routers_per_group #starting gid for routers in dest group + dest_id = dest_group_offset + dest_spinal_offset + num_leaf_routers #gid of destination router + + Ag[source_id,dest_id] = 1 + interlinks.append((source_id, dest_id, source_group_id, dest_group_id)) + + + interlinks.sort(key=lambda x: x[0]) + + for link in interlinks: + fd.write(struct.pack("2i",link[0], link[1])) + print("INTER %d %d srcg %d destg %d" % (link[0], link[1], link[2], link[3]) ) + + for row in Ag: + row_sum = sum(row) + if row_sum > ind_up_radix: + raise Exception("Error: connectivity exceeds radix!") + + for col in Ag.T: + col_sum = sum(col) + if col_sum > ind_up_radix: + raise Exception("Error: Connectivity exceeds radix!") + + else: + raise Exception("Error: Invalid topology size given. Please use {0 | 1 | 2}") + + +if __name__ == '__main__': + main() diff --git a/scripts/neil-inter b/scripts/neil-inter new file mode 100644 index 0000000000000000000000000000000000000000..53df357dc6beadcadbeb1830a26e971dbdf7291f Binary files /dev/null and b/scripts/neil-inter differ diff --git a/scripts/neil-inter-large b/scripts/neil-inter-large new file mode 100644 index 0000000000000000000000000000000000000000..cf548c14cc42b4919a6f0dde86f9240e86c1b88b Binary files /dev/null and b/scripts/neil-inter-large differ diff --git a/scripts/neil-intra b/scripts/neil-intra new file mode 100644 index 0000000000000000000000000000000000000000..e7380f593753716fd108de8c3734a8800ca666a0 Binary files /dev/null and b/scripts/neil-intra differ diff --git a/scripts/neil-intra-large b/scripts/neil-intra-large new file mode 100644 index 0000000000000000000000000000000000000000..6269a7e10dc0f37e1276ff5101853aa2b6ee5311 Binary files /dev/null and b/scripts/neil-intra-large differ diff --git a/src/Makefile.subdir b/src/Makefile.subdir index 20877dbcd1badaa7657dc69aa74155b08cbc8be0..a9c70c2e39c88badee2787b45f31dd3b1b1c4fc3 100644 --- a/src/Makefile.subdir +++ b/src/Makefile.subdir @@ -157,6 +157,7 @@ src_libcodes_la_SOURCES = \ src/networks/model-net/express-mesh.C \ src/networks/model-net/dragonfly.c \ src/networks/model-net/dragonfly-custom.C \ + src/networks/model-net/dragonfly-plus.C \ src/networks/model-net/slimfly.c \ src/networks/model-net/fattree.c \ src/networks/model-net/loggp.c \ @@ -164,7 +165,7 @@ src_libcodes_la_SOURCES = \ src/networks/model-net/model-net-lp.c \ src/networks/model-net/model-net-sched.c \ src/networks/model-net/model-net-sched-impl.h \ - src/networks/model-net/model-net-sched-impl.c + src/networks/model-net/model-net-sched-impl.c src_libcodes_mpi_replay_la_SOURCES = \ src/network-workloads/model-net-mpi-replay.c @@ -190,18 +191,21 @@ bin_PROGRAMS += src/network-workloads/model-net-synthetic bin_PROGRAMS += src/network-workloads/model-net-synthetic-custom-dfly bin_PROGRAMS += src/network-workloads/model-net-synthetic-slimfly bin_PROGRAMS += src/network-workloads/model-net-synthetic-fattree +bin_PROGRAMS += src/network-workloads/model-net-synthetic-dfly-plus + src_workload_codes_workload_dump_SOURCES = \ src/workload/codes-workload-dump.c -src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c +src_network_workloads_model_net_dumpi_traces_dump_SOURCES = src/network-workloads/model-net-dumpi-traces-dump.c src_network_workloads_model_net_synthetic_slimfly_SOURCES = src/network-workloads/model-net-synthetic-slimfly.c src_network_workloads_model_net_mpi_replay_SOURCES = \ src/network-workloads/model-net-mpi-replay.c \ src/network-workloads/model-net-mpi-replay-main.c src_network_workloads_model_net_mpi_replay_CFLAGS = $(AM_CFLAGS) -src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c -src_network_workloads_model_net_synthetic_custom_dfly_SOURCES = src/network-workloads/model-net-synthetic-custom-dfly.c +src_network_workloads_model_net_synthetic_SOURCES = src/network-workloads/model-net-synthetic.c +src_network_workloads_model_net_synthetic_custom_dfly_SOURCES = src/network-workloads/model-net-synthetic-custom-dfly.c +src_network_workloads_model_net_synthetic_dfly_plus_SOURCES = src/network-workloads/model-net-synthetic-dfly-plus.c src_networks_model_net_topology_test_SOURCES = src/networks/model-net/topology-test.c #bin_PROGRAMS += src/network-workload/codes-nw-test diff --git a/src/network-workloads/conf/dragonfly-plus/34kminimal.txt b/src/network-workloads/conf/dragonfly-plus/34kminimal.txt new file mode 100644 index 0000000000000000000000000000000000000000..632aa2d39460dd91f87a9fe652d27d60260f258f --- /dev/null +++ b/src/network-workloads/conf/dragonfly-plus/34kminimal.txt @@ -0,0 +1,74 @@ + : Running Time = 22.1632 seconds + +TW Library Statistics: + Total Events Processed 27882763 + Events Aborted (part of RBs) 0 + Events Rolled Back 1623745 + Event Ties Detected in PE Queues 0 + Efficiency 93.82 % + Total Remote (shared mem) Events Processed 0 + Percent Remote Events 0.00 % + Total Remote (network) Events Processed 2239846 + Percent Remote Events 8.53 % + + Total Roll Backs 213747 + Primary Roll Backs 206428 + Secondary Roll Backs 7319 + Fossil Collect Attempts 108948 + Total GVT Computations 27237 + + Net Events Processed 26259018 + Event Rate (events/sec) 1184800.8 + Total Events Scheduled Past End Time 0 + +TW Memory Statistics: + Events Allocated 4510545 + Memory Allocated 3342712 + Memory Wasted 222 + +TW Network Statistics: + Remote sends 2517896 + Remote recvs 2517896 + +TW Data Structure sizes in bytes (sizeof): + PE struct 624 + KP struct 144 + LP struct 128 + LP Model struct 32 + LP RNGs 80 + Total LP 240 + Event struct 144 + Event struct with Model 752 + +TW Clock Cycle Statistics (MAX values in secs at 1.0000 GHz): + Priority Queue (enq/deq) 1.2060 + AVL Tree (insert/delete) 6.1629 + LZ4 (de)compression 0.0000 + Buddy system 0.0000 + RIO Loading 0.0000 + RIO LP Init 0.1464 + Event Processing 64.3321 + Event Cancel 0.0668 + Event Abort 0.0000 + + GVT 8.9914 + Fossil Collect 4.3308 + Primary Rollbacks 1.8111 + Network Read 10.4349 + Statistics Computation 0.0000 + Statistics Write 0.0000 + Total Time (Note: Using Running Time above for Speedup) 88.8275 + +TW GVT Statistics: MPI AllReduce + GVT Interval 16 + GVT Real Time Interval (cycles) 0 + GVT Real Time Interval (sec) 0.00000000 + Batch Size 16 + + Forced GVT 0 + Total GVT Computations 27237 + Total All Reduce Calls 80374 + Average Reduction / GVT 2.95 + Average number of router hops traversed: 3.967510; average chunk latency: 3.983703 us; maximum chunk latency: 23.508762 us; avg message size: 2048.000000 bytes; finished messages: 675818; finished chunks: 1351636 + + Total packets generated: 1351636; finished: 1351636 \ No newline at end of file diff --git a/src/network-workloads/conf/dragonfly-plus/34kotfa.txt b/src/network-workloads/conf/dragonfly-plus/34kotfa.txt new file mode 100644 index 0000000000000000000000000000000000000000..a9086c2f0887910a273f3dda403627f692c3f2a0 --- /dev/null +++ b/src/network-workloads/conf/dragonfly-plus/34kotfa.txt @@ -0,0 +1,78 @@ + : Running Time = 32.7864 seconds + +TW Library Statistics: + Total Events Processed 33923697 + Events Aborted (part of RBs) 0 + Events Rolled Back 2115066 + Event Ties Detected in PE Queues 0 + Efficiency 93.35 % + Total Remote (shared mem) Events Processed 0 + Percent Remote Events 0.00 % + Total Remote (network) Events Processed 3197222 + Percent Remote Events 10.05 % + + Total Roll Backs 298872 + Primary Roll Backs 293249 + Secondary Roll Backs 5623 + Fossil Collect Attempts 132536 + Total GVT Computations 33134 + + Net Events Processed 31808631 + Event Rate (events/sec) 970177.4 + Total Events Scheduled Past End Time 0 + +TW Memory Statistics: + Events Allocated 4510545 + Memory Allocated 3342712 + Memory Wasted 222 + +TW Network Statistics: + Remote sends 3630262 + Remote recvs 3630262 + +TW Data Structure sizes in bytes (sizeof): + PE struct 624 + KP struct 144 + LP struct 128 + LP Model struct 32 + LP RNGs 80 + Total LP 240 + Event struct 144 + Event struct with Model 752 + +TW Clock Cycle Statistics (MAX values in secs at 1.0000 GHz): + Priority Queue (enq/deq) 1.6238 + AVL Tree (insert/delete) 9.6388 + LZ4 (de)compression 0.0000 + Buddy system 0.0000 + RIO Loading 0.0000 + RIO LP Init 0.1587 + Event Processing 96.7147 + Event Cancel 0.0741 + Event Abort 0.0000 + + GVT 12.5980 + Fossil Collect 5.9422 + Primary Rollbacks 2.6389 + Network Read 15.3961 + Statistics Computation 0.0000 + Statistics Write 0.0000 + Total Time (Note: Using Running Time above for Speedup) 131.4039 + +TW GVT Statistics: MPI AllReduce + GVT Interval 16 + GVT Real Time Interval (cycles) 0 + GVT Real Time Interval (sec) 0.00000000 + Batch Size 16 + + Forced GVT 0 + Total GVT Computations 33134 + Total All Reduce Calls 98239 + Average Reduction / GVT 2.96 + Average number of router hops traversed: 5.336127; average chunk latency: 8.764690 us; maximum chunk latency: 28.623821 us; avg message size: 2048.000000 bytes; finished messages: 675818; finished chunks: 1351636 + + Total packets generated: 1351636; finished: 1351636 + + + + diff --git a/src/network-workloads/conf/dragonfly-plus/dfp-test-inter b/src/network-workloads/conf/dragonfly-plus/dfp-test-inter new file mode 100644 index 0000000000000000000000000000000000000000..46f3df30887525ac7ad640691481bfaa1e44dc08 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/dfp-test-inter differ diff --git a/src/network-workloads/conf/dragonfly-plus/dfp-test-intra b/src/network-workloads/conf/dragonfly-plus/dfp-test-intra new file mode 100644 index 0000000000000000000000000000000000000000..e7380f593753716fd108de8c3734a8800ca666a0 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/dfp-test-intra differ diff --git a/src/network-workloads/conf/dragonfly-plus/dfp-test.conf b/src/network-workloads/conf/dragonfly-plus/dfp-test.conf new file mode 100644 index 0000000000000000000000000000000000000000..54047e6ed9b8c45241f2531b754a170085880213 --- /dev/null +++ b/src/network-workloads/conf/dragonfly-plus/dfp-test.conf @@ -0,0 +1,62 @@ +LPGROUPS +{ + MODELNET_GRP + { + repetitions="5"; +# name of this lp changes according to the model + nw-lp="16"; +# these lp names will be the same for dragonfly-custom model + modelnet_dragonfly_plus="16"; + modelnet_dragonfly_plus_router="8"; + } +} +PARAMS +{ +# packet size in the network + packet_size="1024"; + modelnet_order=( "dragonfly_plus","dragonfly_plus_router" ); + # scheduler options + modelnet_scheduler="fcfs"; +# chunk size in the network (when chunk size = packet size, packets will not be +# divided into chunks) + chunk_size="1024"; + # modelnet_scheduler="round-robin"; + # number of routers within each group + # each router row corresponds to a chassis in Cray systems + num_router_spine="4"; + # each router column corresponds to a slot in a chassis + num_router_leaf="4"; + # number of links connecting between group levels per router + num_level_chans="1"; + # number of groups in the network + num_groups="5"; +# buffer size in bytes for local virtual channels + local_vc_size="8192"; +#buffer size in bytes for global virtual channels + global_vc_size="16384"; +#buffer size in bytes for compute node virtual channels + cn_vc_size="8192"; +#bandwidth in GiB/s for local channels + local_bandwidth="5.25"; +# bandwidth in GiB/s for global channels + global_bandwidth="1.5"; +# bandwidth in GiB/s for compute node-router channels + cn_bandwidth="8.0"; +# ROSS message size + message_size="608"; +# number of compute nodes connected to router, dictated by dragonfly config +# file + num_cns_per_router="4"; +# number of global channels per router + num_global_connections="4"; +# network config file for intra-group connections + intra-group-connections="../src/network-workloads/conf/dragonfly-plus/dfp-test-intra"; +# network config file for inter-group connections + inter-group-connections="../src/network-workloads/conf/dragonfly-plus/dfp-test-inter"; +# routing protocol to be used +# routing="minimal"; + routing="on-the-fly-adaptive"; +# route scoring protocol to be used - options are 'alpha' or 'beta' + route_scoring_metric="beta"; + +} diff --git a/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus-med-34k.conf b/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus-med-34k.conf new file mode 100644 index 0000000000000000000000000000000000000000..33bbd21ecadaac4490712aa24f24755df707ce92 --- /dev/null +++ b/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus-med-34k.conf @@ -0,0 +1,62 @@ +LPGROUPS +{ + MODELNET_GRP + { + repetitions="33"; +# name of this lp changes according to the model + nw-lp="1024"; +# these lp names will be the same for dragonfly-custom model + modelnet_dragonfly_plus="1024"; + modelnet_dragonfly_plus_router="64"; + } +} +PARAMS +{ +# packet size in the network + packet_size="1024"; + modelnet_order=( "dragonfly_plus","dragonfly_plus_router" ); + # scheduler options + modelnet_scheduler="fcfs"; +# chunk size in the network (when chunk size = packet size, packets will not be +# divided into chunks) + chunk_size="1024"; + # modelnet_scheduler="round-robin"; + # number of routers within each group + # each router row corresponds to a chassis in Cray systems + num_router_spine="32"; + # each router column corresponds to a slot in a chassis + num_router_leaf="32"; + # number of links connecting between group levels per router + num_level_chans="1"; + # number of groups in the network + num_groups="33"; + +# predefined threshold (T) deciding when to reassign packet to a lower priority queue + queue_threshold="50"; + +# buffer size in bytes for local virtual channels + local_vc_size="8192"; +#buffer size in bytes for global virtual channels + global_vc_size="16384"; +#buffer size in bytes for compute node virtual channels + cn_vc_size="8192"; +#bandwidth in GiB/s for local channels + local_bandwidth="5.25"; +# bandwidth in GiB/s for global channels + global_bandwidth="1.5"; +# bandwidth in GiB/s for compute node-router channels + cn_bandwidth="8.0"; +# ROSS message size + message_size="608"; +# number of compute nodes connected to router, dictated by dragonfly config +# file + num_cns_per_router="32"; +# number of global ports per router + num_global_connections="32"; +# network config file for intra-group connections + intra-group-connections="../src/network-workloads/conf/dragonfly-plus/neil-34k-intra"; +# network config file for inter-group connections + inter-group-connections="../src/network-workloads/conf/dragonfly-plus/neil-34k-inter"; +# routing protocol to be used + routing="on-the-fly-adaptive"; +} diff --git a/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus-testing-large.conf b/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus-testing-large.conf new file mode 100644 index 0000000000000000000000000000000000000000..8b745d6b20afdfe4460513f880ca612c4e84321b --- /dev/null +++ b/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus-testing-large.conf @@ -0,0 +1,62 @@ +LPGROUPS +{ + MODELNET_GRP + { + repetitions="5"; +# name of this lp changes according to the model + nw-lp="4"; +# these lp names will be the same for dragonfly-custom model + modelnet_dragonfly_plus="4"; + modelnet_dragonfly_plus_router="4"; + } +} +PARAMS +{ +# packet size in the network + packet_size="1024"; + modelnet_order=( "dragonfly_plus","dragonfly_plus_router" ); + # scheduler options + modelnet_scheduler="fcfs"; +# chunk size in the network (when chunk size = packet size, packets will not be +# divided into chunks) + chunk_size="1024"; + # modelnet_scheduler="round-robin"; + # number of routers within each group + # each router row corresponds to a chassis in Cray systems + num_router_spine="2"; + # each router column corresponds to a slot in a chassis + num_router_leaf="2"; + # number of links connecting between group levels per router + num_level_chans="1"; + # number of groups in the network + num_groups="5"; + +# predefined threshold (T) deciding when to reassign packet to a lower priority queue + queue_threshold="50"; + +# buffer size in bytes for local virtual channels + local_vc_size="8192"; +#buffer size in bytes for global virtual channels + global_vc_size="16384"; +#buffer size in bytes for compute node virtual channels + cn_vc_size="8192"; +#bandwidth in GiB/s for local channels + local_bandwidth="5.25"; +# bandwidth in GiB/s for global channels + global_bandwidth="1.5"; +# bandwidth in GiB/s for compute node-router channels + cn_bandwidth="8.0"; +# ROSS message size + message_size="608"; +# number of compute nodes connected to router, dictated by dragonfly config +# file + num_cns_per_router="2"; +# number of global ports per router + num_global_connections="2"; +# network config file for intra-group connections + intra-group-connections="../src/network-workloads/conf/dragonfly-plus/neil-intra-large"; +# network config file for inter-group connections + inter-group-connections="../src/network-workloads/conf/dragonfly-plus/neil-inter-large"; +# routing protocol to be used + routing="non-minimal-leaf"; +} diff --git a/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus.conf b/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus.conf new file mode 100644 index 0000000000000000000000000000000000000000..a636bcb1966f6743054b6d9159f0050bebe73d60 --- /dev/null +++ b/src/network-workloads/conf/dragonfly-plus/modelnet-test-dragonfly-plus.conf @@ -0,0 +1,62 @@ +LPGROUPS +{ + MODELNET_GRP + { + repetitions="12"; +# name of this lp changes according to the model + nw-lp="4"; +# these lp names will be the same for dragonfly-custom model + modelnet_dragonfly_plus="4"; + modelnet_dragonfly_plus_router="2"; + } +} +PARAMS +{ +# packet size in the network + packet_size="1024"; + modelnet_order=( "dragonfly_plus","dragonfly_plus_router" ); + # scheduler options + modelnet_scheduler="fcfs"; +# chunk size in the network (when chunk size = packet size, packets will not be +# divided into chunks) + chunk_size="1024"; + # modelnet_scheduler="round-robin"; + # number of routers within each group + # each router row corresponds to a chassis in Cray systems + num_router_spine="4"; + # each router column corresponds to a slot in a chassis + num_router_leaf="4"; + # number of links connecting between group levels per router + num_level_chans="1"; + # number of groups in the network + num_groups="3"; + +# predefined threshold (T) deciding when to reassign packet to a lower priority queue + queue_threshold="50"; + +# buffer size in bytes for local virtual channels + local_vc_size="8192"; +#buffer size in bytes for global virtual channels + global_vc_size="16384"; +#buffer size in bytes for compute node virtual channels + cn_vc_size="8192"; +#bandwidth in GiB/s for local channels + local_bandwidth="5.25"; +# bandwidth in GiB/s for global channels + global_bandwidth="1.5"; +# bandwidth in GiB/s for compute node-router channels + cn_bandwidth="8.0"; +# ROSS message size + message_size="608"; +# number of compute nodes connected to router, dictated by dragonfly config +# file + num_cns_per_router="4"; +# number of global channels per router + num_global_connections="4"; +# network config file for intra-group connections + intra-group-connections="../src/network-workloads/conf/dragonfly-plus/neil-intra"; +# network config file for inter-group connections + inter-group-connections="../src/network-workloads/conf/dragonfly-plus/neil-inter"; +# routing protocol to be used + routing="minimal"; +} diff --git a/src/network-workloads/conf/dragonfly-plus/neil-34k-inter b/src/network-workloads/conf/dragonfly-plus/neil-34k-inter new file mode 100644 index 0000000000000000000000000000000000000000..c194a302703dd93df01732101973492b2a18cadc Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/neil-34k-inter differ diff --git a/src/network-workloads/conf/dragonfly-plus/neil-34k-intra b/src/network-workloads/conf/dragonfly-plus/neil-34k-intra new file mode 100644 index 0000000000000000000000000000000000000000..c498c5ff36847048fd41cbeb5b6093bdac3ba204 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/neil-34k-intra differ diff --git a/src/network-workloads/conf/dragonfly-plus/neil-inter b/src/network-workloads/conf/dragonfly-plus/neil-inter new file mode 100644 index 0000000000000000000000000000000000000000..53df357dc6beadcadbeb1830a26e971dbdf7291f Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/neil-inter differ diff --git a/src/network-workloads/conf/dragonfly-plus/neil-inter-large b/src/network-workloads/conf/dragonfly-plus/neil-inter-large new file mode 100644 index 0000000000000000000000000000000000000000..ee670fa1e0e95264354d3ecc36e6fa10400e1171 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/neil-inter-large differ diff --git a/src/network-workloads/conf/dragonfly-plus/neil-intra b/src/network-workloads/conf/dragonfly-plus/neil-intra new file mode 100644 index 0000000000000000000000000000000000000000..e7380f593753716fd108de8c3734a8800ca666a0 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/neil-intra differ diff --git a/src/network-workloads/conf/dragonfly-plus/neil-intra-large b/src/network-workloads/conf/dragonfly-plus/neil-intra-large new file mode 100644 index 0000000000000000000000000000000000000000..6269a7e10dc0f37e1276ff5101853aa2b6ee5311 Binary files /dev/null and b/src/network-workloads/conf/dragonfly-plus/neil-intra-large differ diff --git a/src/network-workloads/model-net-synthetic-dfly-plus.c b/src/network-workloads/model-net-synthetic-dfly-plus.c new file mode 100644 index 0000000000000000000000000000000000000000..7db8f43bf49b32e1c0b8d45ec1b6c3d779fff21b --- /dev/null +++ b/src/network-workloads/model-net-synthetic-dfly-plus.c @@ -0,0 +1,465 @@ +/* + * Copyright (C) 2015 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +/* +* The test program generates some synthetic traffic patterns for the model-net network models. +* currently it only support the dragonfly network model uniform random and nearest neighbor traffic patterns. +*/ + +#include "codes/model-net.h" +#include "codes/lp-io.h" +#include "codes/codes.h" +#include "codes/codes_mapping.h" +#include "codes/configuration.h" +#include "codes/lp-type-lookup.h" + +#define PAYLOAD_SZ 2048 + +static int net_id = 0; +static int traffic = 1; +static double arrival_time = 1000.0; + +/* whether to pull instead of push */ +static int num_servers_per_rep = 0; +static int num_routers_per_grp = 0; +static int num_nodes_per_grp = 0; +static int num_nodes_per_cn = 0; +static int num_groups = 0; +static unsigned long long num_nodes = 0; + +static char lp_io_dir[256] = {'\0'}; +static lp_io_handle io_handle; +static unsigned int lp_io_use_suffix = 0; +static int do_lp_io = 0; +static int num_msgs = 20; +static tw_stime sampling_interval = 800000; +static tw_stime sampling_end_time = 1600000; + +typedef struct svr_msg svr_msg; +typedef struct svr_state svr_state; + +/* global variables for codes mapping */ +static char group_name[MAX_NAME_LENGTH]; +static char lp_type_name[MAX_NAME_LENGTH]; +static int group_index, lp_type_index, rep_id, offset; + +/* type of events */ +enum svr_event +{ + KICKOFF, /* kickoff event */ + REMOTE, /* remote event */ + LOCAL /* local event */ +}; + +/* type of synthetic traffic */ +enum TRAFFIC +{ + UNIFORM = 1, /* sends message to a randomly selected node */ + NEAREST_GROUP = 2, /* sends message to the node connected to the neighboring router */ + NEAREST_NEIGHBOR = 3, /* sends message to the next node (potentially connected to the same router) */ + RANDOM_OTHER_GROUP = 4 +}; + +struct svr_state +{ + int msg_sent_count; /* requests sent */ + int msg_recvd_count; /* requests recvd */ + int local_recvd_count; /* number of local messages received */ + tw_stime start_ts; /* time that we started sending requests */ + tw_stime end_ts; /* time that we ended sending requests */ +}; + +struct svr_msg +{ + enum svr_event svr_event_type; + tw_lpid src; /* source of this request or ack */ + int incremented_flag; /* helper for reverse computation */ + model_net_event_return event_rc; +}; + +static void svr_init( + svr_state * ns, + tw_lp * lp); +static void svr_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp); +static void svr_rev_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp); +static void svr_finalize( + svr_state * ns, + tw_lp * lp); + +tw_lptype svr_lp = { + (init_f) svr_init, + (pre_run_f) NULL, + (event_f) svr_event, + (revent_f) svr_rev_event, + (commit_f) NULL, + (final_f) svr_finalize, + (map_f) codes_mapping, + sizeof(svr_state), +}; + +const tw_optdef app_opt [] = +{ + TWOPT_GROUP("Model net synthetic traffic " ), + TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "), + TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "), + TWOPT_STIME("sampling-interval", sampling_interval, "the sampling interval "), + TWOPT_STIME("sampling-end-time", sampling_end_time, "sampling end time "), + TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"), + TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"), + TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"), + TWOPT_END() +}; + +const tw_lptype* svr_get_lp_type() +{ + return(&svr_lp); +} + +static void svr_add_lp_type() +{ + lp_type_register("nw-lp", svr_get_lp_type()); +} + +static void issue_event( + svr_state * ns, + tw_lp * lp) +{ + (void)ns; + tw_event *e; + svr_msg *m; + tw_stime kickoff_time; + + /* each server sends a dummy event to itself that will kick off the real + * simulation + */ + + /* skew each kickoff event slightly to help avoid event ties later on */ + kickoff_time = 1.1 * g_tw_lookahead + tw_rand_exponential(lp->rng, arrival_time); + + e = tw_event_new(lp->gid, kickoff_time, lp); + m = tw_event_data(e); + m->svr_event_type = KICKOFF; + tw_event_send(e); +} + +static void svr_init( + svr_state * ns, + tw_lp * lp) +{ + ns->start_ts = 0.0; + + issue_event(ns, lp); + return; +} + +static void handle_kickoff_rev_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + if(m->incremented_flag) + return; + + if(b->c1) + tw_rand_reverse_unif(lp->rng); + + if(traffic == RANDOM_OTHER_GROUP) { + tw_rand_reverse_unif(lp->rng); + tw_rand_reverse_unif(lp->rng); + } + + model_net_event_rc2(lp, &m->event_rc); + ns->msg_sent_count--; + tw_rand_reverse_unif(lp->rng); +} +static void handle_kickoff_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + if(ns->msg_sent_count >= num_msgs) + { + m->incremented_flag = 1; + return; + } + + m->incremented_flag = 0; + + char anno[MAX_NAME_LENGTH]; + tw_lpid local_dest = -1, global_dest = -1; + + svr_msg * m_local = malloc(sizeof(svr_msg)); + svr_msg * m_remote = malloc(sizeof(svr_msg)); + + m_local->svr_event_type = LOCAL; + m_local->src = lp->gid; + + memcpy(m_remote, m_local, sizeof(svr_msg)); + m_remote->svr_event_type = REMOTE; + + assert(net_id == DRAGONFLY || net_id == DRAGONFLY_PLUS || net_id == DRAGONFLY_CUSTOM); /* only supported for dragonfly model right now. */ + ns->start_ts = tw_now(lp); + codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, anno, &rep_id, &offset); + int local_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + + /* in case of uniform random traffic, send to a random destination. */ + if(traffic == UNIFORM) + { + b->c1 = 1; + local_dest = tw_rand_integer(lp->rng, 0, num_nodes - 1); + } + else if(traffic == NEAREST_GROUP) + { + local_dest = (local_id + num_nodes_per_grp) % num_nodes; + //printf("\n LP %ld sending to %ld num nodes %d ", local_id, local_dest, num_nodes); + } + else if(traffic == NEAREST_NEIGHBOR) + { + local_dest = (local_id + 1) % num_nodes; + // printf("\n LP %ld sending to %ld num nodes %d ", rep_id * 2 + offset, local_dest, num_nodes); + } + else if(traffic == RANDOM_OTHER_GROUP) + { + int my_group_id = local_id / num_nodes_per_grp; + + int other_groups[num_groups-1]; + int added =0; + for(int i = 0; i < num_groups; i++) + { + if(i != my_group_id){ + other_groups[added] = i; + added++; + } + } + int rand_group = other_groups[tw_rand_integer(lp->rng,0,added -1)]; + int rand_node_intra_id = tw_rand_integer(lp->rng, 0, num_nodes_per_grp-1); + + local_dest = (rand_group * num_nodes_per_grp) + rand_node_intra_id; + // printf("\n LP %ld sending to %ld num nodes %d ", local_id, local_dest, num_nodes); + } + assert(local_dest < num_nodes); +// codes_mapping_get_lp_id(group_name, lp_type_name, anno, 1, local_dest / num_servers_per_rep, local_dest % num_servers_per_rep, &global_dest); + global_dest = codes_mapping_get_lpid_from_relative(local_dest, group_name, lp_type_name, NULL, 0); + ns->msg_sent_count++; + model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp); + + issue_event(ns, lp); + return; +} + +static void handle_remote_rev_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + (void)b; + (void)m; + (void)lp; + ns->msg_recvd_count--; +} + +static void handle_remote_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + (void)b; + (void)m; + (void)lp; + ns->msg_recvd_count++; +} + +static void handle_local_rev_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + (void)b; + (void)m; + (void)lp; + ns->local_recvd_count--; +} + +static void handle_local_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + (void)b; + (void)m; + (void)lp; + ns->local_recvd_count++; +} +/* convert ns to seconds */ +static tw_stime ns_to_s(tw_stime ns) +{ + return(ns / (1000.0 * 1000.0 * 1000.0)); +} + +/* convert seconds to ns */ +static tw_stime s_to_ns(tw_stime ns) +{ + return(ns * (1000.0 * 1000.0 * 1000.0)); +} + +static void svr_finalize( + svr_state * ns, + tw_lp * lp) +{ + ns->end_ts = tw_now(lp); + + printf("server %llu recvd %d bytes in %f seconds, %f MiB/s sent_count %d recvd_count %d local_count %d \n", (unsigned long long)lp->gid, PAYLOAD_SZ*ns->msg_recvd_count, ns_to_s(ns->end_ts-ns->start_ts), + ((double)(PAYLOAD_SZ*ns->msg_sent_count)/(double)(1024*1024)/ns_to_s(ns->end_ts-ns->start_ts)), ns->msg_sent_count, ns->msg_recvd_count, ns->local_recvd_count); + return; +} + +static void svr_rev_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + switch (m->svr_event_type) + { + case REMOTE: + handle_remote_rev_event(ns, b, m, lp); + break; + case LOCAL: + handle_local_rev_event(ns, b, m, lp); + break; + case KICKOFF: + handle_kickoff_rev_event(ns, b, m, lp); + break; + default: + assert(0); + break; + } +} + +static void svr_event( + svr_state * ns, + tw_bf * b, + svr_msg * m, + tw_lp * lp) +{ + switch (m->svr_event_type) + { + case REMOTE: + handle_remote_event(ns, b, m, lp); + break; + case LOCAL: + handle_local_event(ns, b, m, lp); + break; + case KICKOFF: + handle_kickoff_event(ns, b, m, lp); + break; + default: + printf("\n Invalid message type %d ", m->svr_event_type); + assert(0); + break; + } +} + +int main( + int argc, + char **argv) +{ + int nprocs; + int rank; + int num_nets; + int *net_ids; + int num_routers; + int num_router_leaf, num_router_spine; + + tw_opt_add(app_opt); + tw_init(&argc, &argv); + + if(argc < 2) + { + printf("\n Usage: mpirun --sync=2/3 mapping_file_name.conf (optional --nkp) "); + MPI_Finalize(); + return 0; + } + + MPI_Comm_rank(MPI_COMM_CODES, &rank); + MPI_Comm_size(MPI_COMM_CODES, &nprocs); + + configuration_load(argv[2], MPI_COMM_CODES, &config); + + model_net_register(); + svr_add_lp_type(); + + codes_mapping_setup(); + + net_ids = model_net_configure(&num_nets); + //assert(num_nets==1); + net_id = *net_ids; + free(net_ids); + + /* 5 days of simulation time */ + g_tw_ts_end = s_to_ns(5 * 24 * 60 * 60); + model_net_enable_sampling(sampling_interval, sampling_end_time); + + if(!(net_id == DRAGONFLY || net_id == DRAGONFLY_PLUS || net_id == DRAGONFLY_CUSTOM)) + { + printf("\n The test works with dragonfly model configuration only! %d %d ", DRAGONFLY_PLUS, net_id); + MPI_Finalize(); + return 0; + } + num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "nw-lp", + NULL, 1); + configuration_get_value_int(&config, "PARAMS", "num_router_leaf", NULL, &num_router_leaf); + configuration_get_value_int(&config, "PARAMS", "num_router_spine", NULL, &num_router_spine); + configuration_get_value_int(&config, "PARAMS", "num_routers", NULL, &num_routers); + configuration_get_value_int(&config, "PARAMS", "num_groups", NULL, &num_groups); + configuration_get_value_int(&config, "PARAMS", "num_cns_per_router", NULL, &num_nodes_per_cn); + + num_routers_per_grp = num_routers; + + num_nodes = num_groups * num_router_leaf * num_nodes_per_cn; + num_nodes_per_grp = num_router_leaf * num_nodes_per_cn; + + assert(num_nodes); + + if(lp_io_dir[0]) + { + do_lp_io = 1; + int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0; + int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES); + assert(ret == 0 || !"lp_io_prepare failure"); + } + tw_run(); + if (do_lp_io){ + int ret = lp_io_flush(io_handle, MPI_COMM_CODES); + assert(ret == 0 || !"lp_io_flush failure"); + } + model_net_report_stats(net_id); + tw_end(); + return 0; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ft=c ts=8 sts=4 sw=4 expandtab + */ diff --git a/src/networks/model-net/dragonfly-plus.C b/src/networks/model-net/dragonfly-plus.C new file mode 100644 index 0000000000000000000000000000000000000000..acaf7243a87ebf63ddfafbec98ee85b36c2ee564 --- /dev/null +++ b/src/networks/model-net/dragonfly-plus.C @@ -0,0 +1,3420 @@ +/* + * Neil McGlohon - Rensselaer Polytechnic Institute + * Original Dragonfly-Custom Base Code by Misbah Mubarak - Argonne National Labs + * + * Copyright (C) 2017 Rensselaer Polytechnic Institute. + * See COPYRIGHT notice in top-level directory. + * + */ + +#include + +#define DEBUG_LP 892 +#include +#include +#include +#include "codes/codes.h" +#include "codes/codes_mapping.h" +#include "codes/jenkins-hash.h" +#include "codes/model-net-lp.h" +#include "codes/model-net-method.h" +#include "codes/model-net.h" +#include "codes/net/dragonfly-plus.h" +#include "codes/quickhash.h" +#include "codes/rc-stack.h" +#include "sys/file.h" + +#include "codes/connection-manager.h" + +#ifdef ENABLE_CORTEX +#include +#include +#endif + +#define DUMP_CONNECTIONS 0 +#define CREDIT_SIZE 8 +#define DFLY_HASH_TABLE_SIZE 4999 + +// debugging parameters +#define TRACK -1 +#define TRACK_PKT -1 +#define TRACK_MSG -1 +#define DEBUG 0 +#define MAX_STATS 65536 + +#define LP_CONFIG_NM_TERM (model_net_lp_config_names[DRAGONFLY_PLUS]) +#define LP_METHOD_NM_TERM (model_net_method_names[DRAGONFLY_PLUS]) +#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[DRAGONFLY_PLUS_ROUTER]) +#define LP_METHOD_NM_ROUT (model_net_method_names[DRAGONFLY_PLUS_ROUTER]) + +using namespace std; +struct Link +{ + int offset; +}; +struct bLink +{ + int offset, dest; +}; + +/*MM: Maintains a list of routers connecting the source and destination groups */ +static vector< vector< vector< int > > > connectionList; + +static vector< ConnectionManager > connManagerList; + +struct IntraGroupLink +{ + int src, dest; +}; + +struct InterGroupLink +{ + int src, dest; +}; + +#ifdef ENABLE_CORTEX +/* This structure is defined at the end of the file */ +extern "C" { +extern cortex_topology dragonfly_plus_cortex_topology; +} +#endif + +static int debug_slot_count = 0; +static long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount; +static long packet_gen = 0, packet_fin = 0; + +static double maxd(double a, double b) +{ + return a < b ? b : a; +} + +/* minimal and non-minimal packet counts for adaptive routing*/ +static int minimal_count = 0, nonmin_count = 0; +static int num_routers_per_mgrp = 0; + +typedef struct dragonfly_plus_param dragonfly_plus_param; +/* annotation-specific parameters (unannotated entry occurs at the + * last index) */ +static uint64_t num_params = 0; +static dragonfly_plus_param *all_params = NULL; +static const config_anno_map_t *anno_map = NULL; + +/* global variables for codes mapping */ +static char lp_group_name[MAX_NAME_LENGTH]; +static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset; + +/* router magic number */ +static int router_magic_num = 0; + +/* terminal magic number */ +static int terminal_magic_num = 0; + +/* Hops within a group */ +static int num_intra_nonmin_hops = 4; +static int num_intra_min_hops = 2; + +static FILE *dragonfly_log = NULL; + +static int sample_bytes_written = 0; +static int sample_rtr_bytes_written = 0; + +static char cn_sample_file[MAX_NAME_LENGTH]; +static char router_sample_file[MAX_NAME_LENGTH]; + +// don't do overhead here - job of MPI layer +static tw_stime mpi_soft_overhead = 0; + +typedef struct terminal_plus_message_list terminal_plus_message_list; +struct terminal_plus_message_list +{ + terminal_plus_message msg; + char *event_data; + terminal_plus_message_list *next; + terminal_plus_message_list *prev; +}; + +static void init_terminal_plus_message_list(terminal_plus_message_list *thisO, terminal_plus_message *inmsg) +{ + thisO->msg = *inmsg; + thisO->event_data = NULL; + thisO->next = NULL; + thisO->prev = NULL; +} + +static void delete_terminal_plus_message_list(void *thisO) +{ + terminal_plus_message_list *toDel = (terminal_plus_message_list *) thisO; + if (toDel->event_data != NULL) + free(toDel->event_data); + free(toDel); +} + +struct dragonfly_plus_param +{ + // configuration parameters + int num_routers; /*Number of routers in a group*/ + double local_bandwidth; /* bandwidth of the router-router channels within a group */ + double global_bandwidth; /* bandwidth of the inter-group router connections */ + double cn_bandwidth; /* bandwidth of the compute node channels connected to routers */ + int num_vcs; /* number of virtual channels */ + int local_vc_size; /* buffer size of the router-router channels */ + int global_vc_size; /* buffer size of the global channels */ + int cn_vc_size; /* buffer size of the compute node channels */ + int chunk_size; /* full-sized packets are broken into smaller chunks.*/ + // derived parameters + int num_cn; + int intra_grp_radix; + + // dfp params start + int num_level_chans; // number of channels between levels of the group(?) + int num_router_spine; // number of spine routers (top level) + int num_router_leaf; // number of leaf routers (bottom level) + int queue_threshold; // predefined queue length threshold T before a packet is routed through a lower priority queue + // dfp params end + + int num_groups; + int radix; + int total_routers; + int total_terminals; + int num_global_connections; + double cn_delay; + double local_delay; + double global_delay; + double credit_delay; + double router_delay; +}; + +struct dfly_hash_key +{ + uint64_t message_id; + tw_lpid sender_id; +}; + +struct dfly_router_sample +{ + tw_lpid router_id; + tw_stime *busy_time; + int64_t *link_traffic_sample; + tw_stime end_time; + long fwd_events; + long rev_events; +}; + +struct dfly_cn_sample +{ + tw_lpid terminal_id; + long fin_chunks_sample; + long data_size_sample; + double fin_hops_sample; + tw_stime fin_chunks_time; + tw_stime busy_time_sample; + tw_stime end_time; + long fwd_events; + long rev_events; +}; + +struct dfly_qhash_entry +{ + struct dfly_hash_key key; + char *remote_event_data; + int num_chunks; + int remote_event_size; + struct qhash_head hash_link; +}; + +/* handles terminal and router events like packet generate/send/receive/buffer */ +typedef struct terminal_state terminal_state; +typedef struct router_state router_state; + +/* dragonfly compute node data structure */ +struct terminal_state +{ + uint64_t packet_counter; + + int packet_gen; + int packet_fin; + + // Dragonfly specific parameters + unsigned int router_id; + unsigned int terminal_id; + + // Each terminal will have an input and output channel with the router + int *vc_occupancy; // NUM_VC + int num_vcs; + tw_stime terminal_available_time; + terminal_plus_message_list **terminal_msgs; + terminal_plus_message_list **terminal_msgs_tail; + int in_send_loop; + struct mn_stats dragonfly_stats_array[CATEGORY_MAX]; + + struct rc_stack *st; + int issueIdle; + int terminal_length; + + const char *anno; + const dragonfly_plus_param *params; + + struct qhash_table *rank_tbl; + uint64_t rank_tbl_pop; + + tw_stime total_time; + uint64_t total_msg_size; + double total_hops; + long finished_msgs; + long finished_chunks; + long finished_packets; + + tw_stime *last_buf_full; + tw_stime busy_time; + + tw_stime max_latency; + tw_stime min_latency; + + char output_buf[4096]; + /* For LP suspend functionality */ + int error_ct; + + /* For sampling */ + long fin_chunks_sample; + long data_size_sample; + double fin_hops_sample; + tw_stime fin_chunks_time; + tw_stime busy_time_sample; + + char sample_buf[4096]; + struct dfly_cn_sample *sample_stat; + int op_arr_size; + int max_arr_size; + + /* for logging forward and reverse events */ + long fwd_events; + long rev_events; +}; + +/* terminal event type (1-4) */ +typedef enum event_t { + T_GENERATE = 1, + T_ARRIVE, + T_SEND, + T_BUFFER, + R_SEND, + R_ARRIVE, + R_BUFFER, +} event_t; + +/* whether the last hop of a packet was global, local or a terminal */ +enum last_hop +{ + GLOBAL = 1, + LOCAL, + TERMINAL, +}; + +typedef enum routing_alg_t +{ + MINIMAL = 1, //will always follow the minimal route from host to host + NON_MINIMAL_SPINE, //will always route through an intermediate spine in an intermediate group for inter group traffic + NON_MINIMAL_LEAF, //will always route through an intermediate leaf in an intermediate group for inter group traffic + ON_THE_FLY_ADAPTIVE, //Choose between Minimal, Nonmin spine, and nonmin leaf at the router level based on own congestion + FULLY_PROG_ADAPTIVE //OTFA with ARNs +} routing_alg_t; + +typedef enum route_scoring_metric_t +{ + ALPHA = 1, //Count queue lengths and pending messages for a port + BETA //Expected Hops to Destination * (Count queue lengths and pending messages) for a port +} route_scoring_metric_t; + +bool isRoutingAdaptive(int alg) +{ + if (alg == ON_THE_FLY_ADAPTIVE || alg == FULLY_PROG_ADAPTIVE) + return true; + else + return false; +} + +bool isRoutingMinimal(int alg) +{ + if (alg == MINIMAL) + return true; + else + return false; +} + +enum LINK_TYPE +{ + GREEN, + BLACK, +}; + +enum router_type +{ + SPINE = 1, + LEAF +}; + +struct router_state +{ + unsigned int router_id; + int group_id; + int op_arr_size; + int max_arr_size; + + router_type dfp_router_type; // Enum to specify whether this router is a spine or a leaf + + ConnectionManager *connMan; + + int *global_channel; + + tw_stime *next_output_available_time; + tw_stime **last_buf_full; + + tw_stime *busy_time; + tw_stime *busy_time_sample; + + terminal_plus_message_list ***pending_msgs; + terminal_plus_message_list ***pending_msgs_tail; + terminal_plus_message_list ***queued_msgs; + terminal_plus_message_list ***queued_msgs_tail; + int *in_send_loop; + int *queued_count; + struct rc_stack *st; + + int **vc_occupancy; + int64_t *link_traffic; + int64_t *link_traffic_sample; + + const char *anno; + const dragonfly_plus_param *params; + + char output_buf[4096]; + char output_buf2[4096]; + + struct dfly_router_sample *rsamples; + + long fwd_events; + long rev_events; +}; + +int dragonfly_plus_get_assigned_router_id(int terminal_id, const dragonfly_plus_param *p); + +static short routing = MINIMAL; +static short scoring = ALPHA; + +static tw_stime dragonfly_total_time = 0; +static tw_stime dragonfly_max_latency = 0; + + +static long long total_hops = 0; +static long long N_finished_packets = 0; +static long long total_msg_sz = 0; +static long long N_finished_msgs = 0; +static long long N_finished_chunks = 0; + +static int dragonfly_rank_hash_compare(void *key, struct qhash_head *link) +{ + struct dfly_hash_key *message_key = (struct dfly_hash_key *) key; + struct dfly_qhash_entry *tmp = NULL; + + tmp = qhash_entry(link, struct dfly_qhash_entry, hash_link); + + if (tmp->key.message_id == message_key->message_id && tmp->key.sender_id == message_key->sender_id) + return 1; + + return 0; +} +static int dragonfly_hash_func(void *k, int table_size) +{ + struct dfly_hash_key *tmp = (struct dfly_hash_key *) k; + uint32_t pc = 0, pb = 0; + bj_hashlittle2(tmp, sizeof(*tmp), &pc, &pb); + return (int) (pc % (table_size - 1)); + /*uint64_t key = (~tmp->message_id) + (tmp->message_id << 18); + key = key * 21; + key = ~key ^ (tmp->sender_id >> 4); + key = key * tmp->sender_id; + return (int)(key & (table_size - 1));*/ +} + +/* convert GiB/s and bytes to ns */ +static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s) +{ + tw_stime time; + + /* bytes to GB */ + time = ((double) bytes) / (1024.0 * 1024.0 * 1024.0); + /* GiB to s */ + time = time / GB_p_s; + /* s to ns */ + time = time * 1000.0 * 1000.0 * 1000.0; + + return (time); +} + +/* returns the dragonfly message size */ +int dragonfly_plus_get_msg_sz(void) +{ + return sizeof(terminal_plus_message); +} + +static void free_tmp(void *ptr) +{ + struct dfly_qhash_entry *dfly = (dfly_qhash_entry *) ptr; + if (dfly->remote_event_data) + free(dfly->remote_event_data); + + if (dfly) + free(dfly); +} + +static void append_to_terminal_plus_message_list(terminal_plus_message_list **thisq, + terminal_plus_message_list **thistail, + int index, + terminal_plus_message_list *msg) +{ + if (thisq[index] == NULL) { + thisq[index] = msg; + } + else { + thistail[index]->next = msg; + msg->prev = thistail[index]; + } + thistail[index] = msg; +} + +static void prepend_to_terminal_plus_message_list(terminal_plus_message_list **thisq, + terminal_plus_message_list **thistail, + int index, + terminal_plus_message_list *msg) +{ + if (thisq[index] == NULL) { + thistail[index] = msg; + } + else { + thisq[index]->prev = msg; + msg->next = thisq[index]; + } + thisq[index] = msg; +} + +static terminal_plus_message_list *return_head(terminal_plus_message_list **thisq, + terminal_plus_message_list **thistail, + int index) +{ + terminal_plus_message_list *head = thisq[index]; + if (head != NULL) { + thisq[index] = head->next; + if (head->next != NULL) { + head->next->prev = NULL; + head->next = NULL; + } + else { + thistail[index] = NULL; + } + } + return head; +} + +static terminal_plus_message_list *return_tail(terminal_plus_message_list **thisq, + terminal_plus_message_list **thistail, + int index) +{ + terminal_plus_message_list *tail = thistail[index]; + assert(tail); + if (tail->prev != NULL) { + tail->prev->next = NULL; + thistail[index] = tail->prev; + tail->prev = NULL; + } + else { + thistail[index] = NULL; + thisq[index] = NULL; + } + return tail; +} + +static void dragonfly_read_config(const char *anno, dragonfly_plus_param *params) +{ + /*Adding init for router magic number*/ + uint32_t h1 = 0, h2 = 0; + bj_hashlittle2(LP_METHOD_NM_ROUT, strlen(LP_METHOD_NM_ROUT), &h1, &h2); + router_magic_num = h1 + h2; + + bj_hashlittle2(LP_METHOD_NM_TERM, strlen(LP_METHOD_NM_TERM), &h1, &h2); + terminal_magic_num = h1 + h2; + + // shorthand + dragonfly_plus_param *p = params; + int myRank; + MPI_Comm_rank(MPI_COMM_CODES, &myRank); + + int rc = configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size); + if (rc) { + p->local_vc_size = 1024; + fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size); + } + + rc = configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size); + if (rc) { + p->global_vc_size = 2048; + fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size); + } + + rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size); + if (rc) { + p->cn_vc_size = 1024; + fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size); + } + + rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size); + if (rc) { + p->chunk_size = 512; + fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size); + } + + rc = configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth); + if (rc) { + p->local_bandwidth = 5.25; + fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth); + } + + rc = configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth); + if (rc) { + p->global_bandwidth = 4.7; + fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth); + } + + rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth); + if (rc) { + p->cn_bandwidth = 5.25; + fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", + p->cn_bandwidth); + } + + rc = configuration_get_value_double(&config, "PARAMS", "router_delay", anno, &p->router_delay); + if (rc) { + p->router_delay = 100; + } + + configuration_get_value(&config, "PARAMS", "cn_sample_file", anno, cn_sample_file, MAX_NAME_LENGTH); + configuration_get_value(&config, "PARAMS", "rt_sample_file", anno, router_sample_file, MAX_NAME_LENGTH); + + char routing_str[MAX_NAME_LENGTH]; + configuration_get_value(&config, "PARAMS", "routing", anno, routing_str, MAX_NAME_LENGTH); + if (strcmp(routing_str, "minimal") == 0) + routing = MINIMAL; + else if (strcmp(routing_str, "non-minimal-spine") == 0) + routing = NON_MINIMAL_SPINE; + else if (strcmp(routing_str, "non-minimal-leaf") == 0) + routing = NON_MINIMAL_LEAF; + else if (strcmp(routing_str, "on-the-fly-adaptive") == 0) + routing = ON_THE_FLY_ADAPTIVE; + else if (strcmp(routing_str, "fully-prog-adaptive") == 0) + routing = FULLY_PROG_ADAPTIVE; + else { + fprintf(stderr, "No routing protocol specified, setting to minimal routing\n"); + routing = MINIMAL; + } + + char scoring_str[MAX_NAME_LENGTH]; + configuration_get_value(&config, "PARAMS", "route_scoring_metric", anno, scoring_str, MAX_NAME_LENGTH); + if (strcmp(scoring_str, "alpha") == 0) + scoring = ALPHA; + else if (strcmp(scoring_str, "beta") == 0) + scoring = BETA; + else { + fprintf(stderr, "No route scoring protocol specified, setting to alpha scoring\n"); + scoring = ALPHA; + } + + /* MM: This should be 2 for dragonfly plus*/ + p->num_vcs = 2; + + rc = configuration_get_value_int(&config, "PARAMS", "num_groups", anno, &p->num_groups); + if (rc) { + printf("Number of groups not specified. Aborting"); + MPI_Abort(MPI_COMM_CODES, 1); + } + rc = configuration_get_value_int(&config, "PARAMS", "num_router_spine", anno, &p->num_router_spine); + if (rc) { + p->num_router_spine = 1; + } + rc = configuration_get_value_int(&config, "PARAMS", "num_router_leaf", anno, &p->num_router_leaf); + if (rc) { + p->num_router_leaf = 1; + } + rc = configuration_get_value_int(&config, "PARAMS", "num_level_chans", anno, &p->num_level_chans); + if (rc) { + printf("\n Number of links connecting chassis not specified, setting to default value 1 "); + p->num_level_chans = 1; + } + + + p->num_routers = p->num_router_spine + p->num_router_leaf; // num routers per group + p->intra_grp_radix = (p->num_routers / 2) * p->num_level_chans; + + + rc = configuration_get_value_int(&config, "PARAMS", "num_cns_per_router", anno, &p->num_cn); + if (rc) { + printf("\n Number of cns per router not specified, setting to %d ", 4); + p->num_cn = 4; + } + + rc = configuration_get_value_int(&config, "PARAMS", "num_global_connections", anno, &p->num_global_connections); + if (rc) { + printf("\n Number of global channels per router not specified, setting to 10 "); + p->num_global_connections = 10; + } + p->radix = p->intra_grp_radix + p->num_global_connections + + p->num_cn; // TODO this may not be sufficient, radix isn't same for leaf and spine routers + p->total_routers = p->num_groups * p->num_routers; + p->total_terminals = (p->num_groups * p->num_router_leaf) * p->num_cn; + + // read intra group connections, store from a router's perspective + // all links to the same router form a vector + char intraFile[MAX_NAME_LENGTH]; + configuration_get_value(&config, "PARAMS", "intra-group-connections", anno, intraFile, MAX_NAME_LENGTH); + if (strlen(intraFile) <= 0) { + tw_error(TW_LOC, "Intra group connections file not specified. Aborting"); + } + + //setup Connection Managers for each router + for(int i = 0; i < p->total_routers; i++) + { + int src_id_global = i; + int src_id_local = i % p->num_routers; + int src_group = i / p->num_routers; + + ConnectionManager conman = ConnectionManager(src_id_local, src_id_global, src_group, p->intra_grp_radix, p->num_global_connections, p->num_cn, p->num_routers); + connManagerList.push_back(conman); + } + + FILE *groupFile = fopen(intraFile, "rb"); + if (!groupFile) + tw_error(TW_LOC, "intra-group file not found "); + + IntraGroupLink newLink; + while (fread(&newLink, sizeof(IntraGroupLink), 1, groupFile) != 0) { + int src_id_local = newLink.src; + int dest_id_local = newLink.dest; + for(int i = 0; i < p->total_routers; i++) + { + int group_id = i/p->num_routers; + if (i % p->num_routers == src_id_local) + { + int dest_id_global = group_id * p->num_routers + dest_id_local; + connManagerList[i].add_connection(dest_id_global, CONN_LOCAL); + } + } + } + fclose(groupFile); + + //terminal assignment! + for(int i = 0; i < p->total_terminals; i++) + { + int assigned_router_id = dragonfly_plus_get_assigned_router_id(i, p); + int assigned_group_id = assigned_router_id / p->num_routers; + connManagerList[assigned_router_id].add_connection(i, CONN_TERMINAL); + } + + // read inter group connections, store from a router's perspective + // also create a group level table that tells all the connecting routers + char interFile[MAX_NAME_LENGTH]; + configuration_get_value(&config, "PARAMS", "inter-group-connections", anno, interFile, MAX_NAME_LENGTH); + if (strlen(interFile) <= 0) { + tw_error(TW_LOC, "Inter group connections file not specified. Aborting"); + } + FILE *systemFile = fopen(interFile, "rb"); + if (!myRank) { + printf("Reading inter-group connectivity file: %s\n", interFile); + printf("\nTotal routers: %d; total groups: %d \n", p->total_routers, p->num_groups); + } + + connectionList.resize(p->num_groups); + for (int g = 0; g < connectionList.size(); g++) { + connectionList[g].resize(p->num_groups); + } + + InterGroupLink newInterLink; + while (fread(&newInterLink, sizeof(InterGroupLink), 1, systemFile) != 0) { + int src_id_global = newInterLink.src; + int src_group_id = src_id_global / p->num_routers; + int dest_id_global = newInterLink.dest; + int dest_group_id = dest_id_global / p->num_routers; + + // printf("[%d -> %d]\n",src_id_global, dest_id_global); + connManagerList[src_id_global].add_connection(dest_id_global, CONN_GLOBAL); + + int r; + for (r = 0; r < connectionList[src_group_id][dest_group_id].size(); r++) { + if (connectionList[src_group_id][dest_group_id][r] == newInterLink.src) + break; + } + if (r == connectionList[src_group_id][dest_group_id].size()) { + connectionList[src_group_id][dest_group_id].push_back(newInterLink.src); + } + } + + if (!myRank) { + printf("\n Total nodes %d routers %d groups %d routers per group %d radix %d\n", + p->num_cn * p->num_router_leaf * p->num_groups, p->total_routers, p->num_groups, p->num_routers, p->radix); + } + + p->cn_delay = bytes_to_ns(p->chunk_size, p->cn_bandwidth); + p->local_delay = bytes_to_ns(p->chunk_size, p->local_bandwidth); + p->global_delay = bytes_to_ns(p->chunk_size, p->global_bandwidth); + p->credit_delay = bytes_to_ns(CREDIT_SIZE, p->local_bandwidth); // assume 8 bytes packet +} + +void dragonfly_plus_configure() +{ + anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM_TERM); + assert(anno_map); + num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0); + all_params = (dragonfly_plus_param *) malloc(num_params * sizeof(*all_params)); + + for (int i = 0; i < anno_map->num_annos; i++) { + const char *anno = anno_map->annotations[i].ptr; + dragonfly_read_config(anno, &all_params[i]); + } + if (anno_map->has_unanno_lp > 0) { + dragonfly_read_config(NULL, &all_params[anno_map->num_annos]); + } +#ifdef ENABLE_CORTEX + model_net_topology = dragonfly_plus_cortex_topology; +#endif +} + +/* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */ +void dragonfly_plus_report_stats() +{ + long long avg_hops, total_finished_packets, total_finished_chunks; + long long total_finished_msgs, final_msg_sz; + tw_stime avg_time, max_time; + int total_minimal_packets, total_nonmin_packets; + long total_gen, total_fin; + + MPI_Reduce(&total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&dragonfly_total_time, &avg_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES); + + MPI_Reduce(&packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES); + MPI_Reduce(&packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES); + // if (routing == ADAPTIVE || routing == PROG_ADAPTIVE) { + // MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES); + // MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES); + // } + + /* print statistics */ + if (!g_tw_mynode) { + printf( + " Average number of router hops traversed: %f; average chunk latency: %lf us; maximum chunk latency: %lf us; " + "avg message size: %lf bytes; finished messages: %lld; finished chunks: %lld \n", + (float) avg_hops / total_finished_chunks, avg_time / (total_finished_chunks * 1000), + max_time / 1000, (float) final_msg_sz / total_finished_msgs, total_finished_msgs, + total_finished_chunks); + // if (routing == ADAPTIVE || routing == PROG_ADAPTIVE) + // printf( + // "\n ADAPTIVE ROUTING STATS: %d chunks routed minimally %d chunks routed non-minimally " + // "completed packets %lld \n", + // total_minimal_packets, total_nonmin_packets, total_finished_chunks); + + printf("\n Total packets generated: %ld; finished: %ld \n", total_gen, total_fin); + } + return; +} + +int dragonfly_plus_get_router_type(int router_id, const dragonfly_plus_param *p) +{ + int num_groups = p->num_groups; + int num_routers = p->num_routers; + int num_router_leaf = p->num_router_leaf; + + int group_id = router_id / num_groups; + int router_local_id = router_id % num_routers; + + if (router_local_id > num_router_leaf) + return SPINE; + else + return LEAF; +} + +/* get the router id associated with a given terminal id */ +int dragonfly_plus_get_assigned_router_id(int terminal_id, const dragonfly_plus_param *p) +{ + // currently supports symmetrical bipartite spine/leaf router configurations + // first half of routers in a given group are leafs which have terminals + // second half of routers in a given group are spines which have no terminals + + int num_groups = p->num_groups; // number of groups of routers in the network + int num_routers = p->num_routers; // num routers per group + int num_router_leaf = p->num_router_leaf; // num leaf routers per group + int num_cn = p->num_cn; // num compute nodes per leaf router + + int group_id = terminal_id / (num_router_leaf * num_cn); + int local_router_id = terminal_id % (num_router_leaf * num_cn) / num_router_leaf; + + int router_id = (group_id * num_routers) + local_router_id; + // printf("Terminal %d: assigned to group %d, local router id %d, global router id %d\n",terminal_id, + // group_id, local_router_id, router_id); + + return router_id; +} + +/* initialize a dragonfly compute node terminal */ +void terminal_plus_init(terminal_state *s, tw_lp *lp) +{ + // printf("%d: Terminal Init()\n",lp->gid); + s->packet_gen = 0; + s->packet_fin = 0; + + int i; + char anno[MAX_NAME_LENGTH]; + + // Assign the global router ID + // TODO: be annotation-aware + codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, anno, + &mapping_rep_id, &mapping_offset); + if (anno[0] == '\0') { + s->anno = NULL; + s->params = &all_params[num_params - 1]; + } + else { + s->anno = strdup(anno); + int id = configuration_get_annotation_index(anno, anno_map); + s->params = &all_params[id]; + } + + int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM, s->anno, 0); + + s->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + s->router_id = dragonfly_plus_get_assigned_router_id(s->terminal_id, s->params); + // s->router_id=(int)s->terminal_id / (s->params->num_cn); //TODO I think this is where the router that + // the terminal is connected to is specified + + // printf("%d gid is TERMINAL %d with assigned router %d\n",lp->gid,s->terminal_id,s->router_id); + s->terminal_available_time = 0.0; + s->packet_counter = 0; + s->min_latency = INT_MAX; + s->max_latency = 0; + + s->finished_msgs = 0; + s->finished_chunks = 0; + s->finished_packets = 0; + s->total_time = 0.0; + s->total_msg_size = 0; + + s->busy_time = 0.0; + + s->fwd_events = 0; + s->rev_events = 0; + + rc_stack_create(&s->st); + s->num_vcs = 1; + s->vc_occupancy = (int *) malloc(s->num_vcs * sizeof(int)); + s->last_buf_full = (tw_stime *) malloc(s->num_vcs * sizeof(tw_stime)); + + for (i = 0; i < s->num_vcs; i++) { + s->last_buf_full[i] = 0.0; + s->vc_occupancy[i] = 0; + } + + + s->rank_tbl = NULL; + s->terminal_msgs = + (terminal_plus_message_list **) malloc(s->num_vcs * sizeof(terminal_plus_message_list *)); + s->terminal_msgs_tail = + (terminal_plus_message_list **) malloc(s->num_vcs * sizeof(terminal_plus_message_list *)); + s->terminal_msgs[0] = NULL; + s->terminal_msgs_tail[0] = NULL; + s->terminal_length = 0; + s->in_send_loop = 0; + s->issueIdle = 0; + + return; +} + +/* sets up the router virtual channels, global channels, + * local channels, compute node channels */ +void router_plus_setup(router_state *r, tw_lp *lp) +{ + // printf("%d: Router Init()\n",lp->gid); + + char anno[MAX_NAME_LENGTH]; + codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, anno, + &mapping_rep_id, &mapping_offset); + + if (anno[0] == '\0') { + r->anno = NULL; + r->params = &all_params[num_params - 1]; + } + else { + r->anno = strdup(anno); + int id = configuration_get_annotation_index(anno, anno_map); + r->params = &all_params[id]; + } + + // shorthand + const dragonfly_plus_param *p = r->params; + + num_routers_per_mgrp = + codes_mapping_get_lp_count(lp_group_name, 1, "modelnet_dragonfly_plus_router", NULL, 0); + int num_grp_reps = codes_mapping_get_group_reps(lp_group_name); + if (p->total_routers != num_grp_reps * num_routers_per_mgrp) + tw_error(TW_LOC, + "\n Config error: num_routers specified %d total routers computed in the network %d " + "does not match with repetitions * dragonfly_router %d ", + p->num_routers, p->total_routers, num_grp_reps * num_routers_per_mgrp); + + r->router_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + r->group_id = r->router_id / p->num_routers; + + // printf("\n Local router id %d global id %d ", r->router_id, lp->gid); + + r->fwd_events = 0; + r->rev_events = 0; + + // Determine if router is a spine or a leaf + int intra_group_id = r->router_id % p->num_routers; + if (intra_group_id >= (p->num_routers / 2)) { //TODO this assumes symmetric spine and leafs + r->dfp_router_type = SPINE; + // printf("%lu: %i is a SPINE\n",lp->gid, r->router_id); + } + else { + r->dfp_router_type = LEAF; + // printf("%lu: %i is a LEAF\n",lp->gid, r->router_id); + } + + r->connMan = &connManagerList[r->router_id]; + + r->global_channel = (int *) malloc(p->num_global_connections * sizeof(int)); + r->next_output_available_time = (tw_stime *) malloc(p->radix * sizeof(tw_stime)); + r->link_traffic = (int64_t *) malloc(p->radix * sizeof(int64_t)); + r->link_traffic_sample = (int64_t *) malloc(p->radix * sizeof(int64_t)); + + r->vc_occupancy = (int **) malloc(p->radix * sizeof(int *)); + r->in_send_loop = (int *) malloc(p->radix * sizeof(int)); + r->pending_msgs = + (terminal_plus_message_list ***) malloc(p->radix * sizeof(terminal_plus_message_list **)); + r->pending_msgs_tail = + (terminal_plus_message_list ***) malloc(p->radix * sizeof(terminal_plus_message_list **)); + r->queued_msgs = + (terminal_plus_message_list ***) malloc(p->radix * sizeof(terminal_plus_message_list **)); + r->queued_msgs_tail = + (terminal_plus_message_list ***) malloc(p->radix * sizeof(terminal_plus_message_list **)); + r->queued_count = (int *) malloc(p->radix * sizeof(int)); + r->last_buf_full = (tw_stime **) malloc(p->radix * sizeof(tw_stime *)); + r->busy_time = (tw_stime *) malloc(p->radix * sizeof(tw_stime)); + r->busy_time_sample = (tw_stime *) malloc(p->radix * sizeof(tw_stime)); + + rc_stack_create(&r->st); + + for (int i = 0; i < p->radix; i++) { + // Set credit & router occupancy + r->busy_time[i] = 0.0; + r->busy_time_sample[i] = 0.0; + r->next_output_available_time[i] = 0; + r->link_traffic[i] = 0; + r->link_traffic_sample[i] = 0; + r->queued_count[i] = 0; + r->in_send_loop[i] = 0; + r->vc_occupancy[i] = (int *) malloc(p->num_vcs * sizeof(int)); + r->pending_msgs[i] = + (terminal_plus_message_list **) malloc(p->num_vcs * sizeof(terminal_plus_message_list *)); + r->last_buf_full[i] = (tw_stime *) malloc(p->num_vcs * sizeof(tw_stime)); + r->pending_msgs_tail[i] = + (terminal_plus_message_list **) malloc(p->num_vcs * sizeof(terminal_plus_message_list *)); + r->queued_msgs[i] = + (terminal_plus_message_list **) malloc(p->num_vcs * sizeof(terminal_plus_message_list *)); + r->queued_msgs_tail[i] = + (terminal_plus_message_list **) malloc(p->num_vcs * sizeof(terminal_plus_message_list *)); + for (int j = 0; j < p->num_vcs; j++) { + r->last_buf_full[i][j] = 0.0; + r->vc_occupancy[i][j] = 0; + r->pending_msgs[i][j] = NULL; + r->pending_msgs_tail[i][j] = NULL; + r->queued_msgs[i][j] = NULL; + r->queued_msgs_tail[i][j] = NULL; + } + } + + return; +} + +/* MM: These packet events (packet_send, packet_receive etc.) will be used as is, basically, the routing + * functions will be changed only. */ +/* dragonfly packet event , generates a dragonfly packet on the compute node */ +static tw_stime dragonfly_plus_packet_event(model_net_request const *req, + uint64_t message_offset, + uint64_t packet_size, + tw_stime offset, + mn_sched_params const *sched_params, + void const *remote_event, + void const *self_event, + tw_lp *sender, + int is_last_pckt) +{ + (void) message_offset; + (void) sched_params; + tw_event *e_new; + tw_stime xfer_to_nic_time; + terminal_plus_message *msg; + char *tmp_ptr; + + xfer_to_nic_time = codes_local_latency(sender); + // e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender); + // msg = tw_event_data(e_new); + e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time + offset, sender, DRAGONFLY_PLUS, + (void **) &msg, (void **) &tmp_ptr); + strcpy(msg->category, req->category); + msg->final_dest_gid = req->final_dest_lp; + msg->total_size = req->msg_size; + msg->sender_lp = req->src_lp; + msg->sender_mn_lp = sender->gid; + msg->packet_size = packet_size; + msg->travel_start_time = tw_now(sender); + msg->remote_event_size_bytes = 0; + msg->local_event_size_bytes = 0; + msg->type = T_GENERATE; + msg->dest_terminal_id = req->dest_mn_lp; + msg->dfp_dest_terminal_id = codes_mapping_get_lp_relative_id(msg->dest_terminal_id,0,0); + msg->message_id = req->msg_id; + msg->is_pull = req->is_pull; + msg->pull_size = req->pull_size; + msg->magic = terminal_magic_num; + msg->msg_start_time = req->msg_start_time; + + if (is_last_pckt) /* Its the last packet so pass in remote and local event information*/ + { + if (req->remote_event_size > 0) { + msg->remote_event_size_bytes = req->remote_event_size; + memcpy(tmp_ptr, remote_event, req->remote_event_size); + tmp_ptr += req->remote_event_size; + } + if (req->self_event_size > 0) { + msg->local_event_size_bytes = req->self_event_size; + memcpy(tmp_ptr, self_event, req->self_event_size); + tmp_ptr += req->self_event_size; + } + } + // printf("\n dragonfly remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, + // msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time); + tw_event_send(e_new); + return xfer_to_nic_time; +} + +/* dragonfly packet event reverse handler */ +static void dragonfly_plus_packet_event_rc(tw_lp *sender) +{ + codes_local_latency_reverse(sender); + return; +} + +/*MM: This will also be used as is. This is meant to sent a credit back to the + * sending router. */ +/*When a packet is sent from the current router and a buffer slot becomes available, a credit is sent back to + * schedule another packet event*/ +static void router_credit_send(router_state *s, terminal_plus_message *msg, tw_lp *lp, int sq) +{ + tw_event *buf_e; + tw_stime ts; + terminal_plus_message *buf_msg; + + int dest = 0, type = R_BUFFER; + int is_terminal = 0; + + const dragonfly_plus_param *p = s->params; + + // Notify sender terminal about available buffer space + if (msg->last_hop == TERMINAL) { + dest = msg->src_terminal_id; + type = T_BUFFER; + is_terminal = 1; + } + else if (msg->last_hop == GLOBAL || msg->last_hop == LOCAL) { + dest = msg->intm_lp_id; + } + else + printf("\n Invalid message type"); + + ts = g_tw_lookahead + p->credit_delay + tw_rand_unif(lp->rng); + + if (is_terminal) { + buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY_PLUS, (void **) &buf_msg, NULL); + buf_msg->magic = terminal_magic_num; + } + else { + buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY_PLUS_ROUTER, (void **) &buf_msg, NULL); + buf_msg->magic = router_magic_num; + } + + if (sq == -1) { + buf_msg->vc_index = msg->vc_index; + buf_msg->output_chan = msg->output_chan; + } + else { + buf_msg->vc_index = msg->saved_vc; + buf_msg->output_chan = msg->saved_channel; + } + + buf_msg->type = type; + + tw_event_send(buf_e); + return; +} + +static void packet_generate_rc(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + s->packet_gen--; + packet_gen--; + + tw_rand_reverse_unif(lp->rng); + + int num_chunks = msg->packet_size / s->params->chunk_size; + if (msg->packet_size < s->params->chunk_size) + num_chunks++; + + int i; + for (i = 0; i < num_chunks; i++) { + delete_terminal_plus_message_list(return_tail(s->terminal_msgs, s->terminal_msgs_tail, 0)); + s->terminal_length -= s->params->chunk_size; + } + if (bf->c5) { + codes_local_latency_reverse(lp); + s->in_send_loop = 0; + } + if (bf->c11) { + s->issueIdle = 0; + if (bf->c8) { + s->last_buf_full[0] = msg->saved_busy_time; + } + } + struct mn_stats *stat; + stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); + stat->send_count--; + stat->send_bytes -= msg->packet_size; + stat->send_time -= (1 / s->params->cn_bandwidth) * msg->packet_size; +} + +/* generates packet at the current dragonfly compute node */ +static void packet_generate(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + packet_gen++; + s->packet_gen++; + + tw_stime ts, nic_ts; + + assert(lp->gid != msg->dest_terminal_id); + const dragonfly_plus_param *p = s->params; + + int total_event_size; + uint64_t num_chunks = msg->packet_size / p->chunk_size; + double cn_delay = s->params->cn_delay; + + if (msg->packet_size < s->params->chunk_size) + num_chunks++; + + if (msg->packet_size < s->params->chunk_size) + cn_delay = bytes_to_ns(msg->packet_size % s->params->chunk_size, s->params->cn_bandwidth); + + nic_ts = g_tw_lookahead + (num_chunks * cn_delay) + tw_rand_unif(lp->rng); + + msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter; + msg->my_N_hop = 0; + msg->my_l_hop = 0; + msg->my_g_hop = 0; + + // if(msg->dest_terminal_id == TRACK) + if (msg->packet_ID == LLU(TRACK_PKT)) + printf("\n Packet %llu generated at terminal %d dest %llu size %llu num chunks %llu ", msg->packet_ID, + s->terminal_id, LLU(msg->dest_terminal_id), LLU(msg->packet_size), LLU(num_chunks)); + + for (int i = 0; i < num_chunks; i++) { + terminal_plus_message_list *cur_chunk = + (terminal_plus_message_list *) malloc(sizeof(terminal_plus_message_list)); + msg->origin_router_id = s->router_id; + init_terminal_plus_message_list(cur_chunk, msg); + + if (msg->remote_event_size_bytes + msg->local_event_size_bytes > 0) { + cur_chunk->event_data = + (char *) malloc(msg->remote_event_size_bytes + msg->local_event_size_bytes); + } + + void *m_data_src = model_net_method_get_edata(DRAGONFLY_PLUS, msg); + if (msg->remote_event_size_bytes) { + memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes); + } + if (msg->local_event_size_bytes) { + m_data_src = (char *) m_data_src + msg->remote_event_size_bytes; + memcpy((char *) cur_chunk->event_data + msg->remote_event_size_bytes, m_data_src, + msg->local_event_size_bytes); + } + + cur_chunk->msg.chunk_id = i; + cur_chunk->msg.origin_router_id = s->router_id; + append_to_terminal_plus_message_list(s->terminal_msgs, s->terminal_msgs_tail, 0, cur_chunk); + s->terminal_length += s->params->chunk_size; + } + + if (s->terminal_length < 2 * s->params->cn_vc_size) { + model_net_method_idle_event(nic_ts, 0, lp); + } + else { + bf->c11 = 1; + s->issueIdle = 1; + + if (s->last_buf_full[0] == 0.0) { + bf->c8 = 1; + msg->saved_busy_time = s->last_buf_full[0]; + /* TODO: Assumes a single vc from terminal to router */ + s->last_buf_full[0] = tw_now(lp); + } + } + + if (s->in_send_loop == 0) { + bf->c5 = 1; + ts = codes_local_latency(lp); + terminal_plus_message *m; + tw_event *e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_PLUS, (void **) &m, NULL); + m->type = T_SEND; + m->magic = terminal_magic_num; + s->in_send_loop = 1; + tw_event_send(e); + } + + total_event_size = + model_net_get_msg_sz(DRAGONFLY_PLUS) + msg->remote_event_size_bytes + msg->local_event_size_bytes; + mn_stats *stat; + stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); + stat->send_count++; + stat->send_bytes += msg->packet_size; + stat->send_time += (1 / p->cn_bandwidth) * msg->packet_size; + if (stat->max_event_size < total_event_size) + stat->max_event_size = total_event_size; + + return; +} + +static void packet_send_rc(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + if (bf->c1) { + s->in_send_loop = 1; + + if (bf->c10) + s->last_buf_full[0] = msg->saved_busy_time; + + return; + } + + tw_rand_reverse_unif(lp->rng); + s->terminal_available_time = msg->saved_available_time; + if (bf->c2) { + codes_local_latency_reverse(lp); + } + + s->terminal_length += s->params->chunk_size; + s->packet_counter--; + s->vc_occupancy[0] -= s->params->chunk_size; + + terminal_plus_message_list *cur_entry = (terminal_plus_message_list *) rc_stack_pop(s->st); + + prepend_to_terminal_plus_message_list(s->terminal_msgs, s->terminal_msgs_tail, 0, cur_entry); + if (bf->c3) { + tw_rand_reverse_unif(lp->rng); + } + if (bf->c4) { + s->in_send_loop = 1; + } + if (bf->c5) { + tw_rand_reverse_unif(lp->rng); + s->issueIdle = 1; + if (bf->c6) { + s->busy_time = msg->saved_total_time; + s->last_buf_full[0] = msg->saved_busy_time; + s->busy_time_sample = msg->saved_sample_time; + } + } + return; +} +/* sends the packet from the current dragonfly compute node to the attached router */ +static void packet_send(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + tw_stime ts; + tw_event *e; + terminal_plus_message *m; + tw_lpid router_id; + + terminal_plus_message_list *cur_entry = s->terminal_msgs[0]; + + if (s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size) { + if (s->last_buf_full[0] == 0.0) { + bf->c10 = 1; + msg->saved_busy_time = s->last_buf_full[0]; + s->last_buf_full[0] = tw_now(lp); + } + } + + if (s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size || cur_entry == NULL) { + bf->c1 = 1; + s->in_send_loop = 0; + return; + } + + uint64_t num_chunks = cur_entry->msg.packet_size / s->params->chunk_size; + if (cur_entry->msg.packet_size < s->params->chunk_size) + num_chunks++; + + tw_stime delay = s->params->cn_delay; + if ((cur_entry->msg.packet_size < s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1)) + delay = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, s->params->cn_bandwidth); + + msg->saved_available_time = s->terminal_available_time; + ts = g_tw_lookahead + delay + tw_rand_unif(lp->rng); + s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp)); + s->terminal_available_time += ts; + + ts = s->terminal_available_time - tw_now(lp); + // TODO: be annotation-aware + codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL, &mapping_type_id, NULL, + &mapping_rep_id, &mapping_offset); + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1, s->router_id / num_routers_per_mgrp, + s->router_id % num_routers_per_mgrp, &router_id); + // printf("\n Local router id %d global router id %d ", s->router_id, router_id); + // we are sending an event to the router, so no method_event here + void *remote_event; + e = model_net_method_event_new(router_id, ts, lp, DRAGONFLY_PLUS_ROUTER, (void **) &m, &remote_event); + memcpy(m, &cur_entry->msg, sizeof(terminal_plus_message)); + if (m->remote_event_size_bytes) { + memcpy(remote_event, cur_entry->event_data, m->remote_event_size_bytes); + } + + m->type = R_ARRIVE; + m->src_terminal_id = lp->gid; + m->vc_index = 0; + m->last_hop = TERMINAL; + m->magic = router_magic_num; + m->path_type = -1; + m->local_event_size_bytes = 0; + m->intm_rtr_id = -1; + m->intm_group_id = -1; + tw_event_send(e); + + + if (cur_entry->msg.chunk_id == num_chunks - 1 && (cur_entry->msg.local_event_size_bytes > 0)) { + bf->c2 = 1; + tw_stime local_ts = codes_local_latency(lp); + tw_event *e_new = tw_event_new(cur_entry->msg.sender_lp, local_ts, lp); + void *m_new = tw_event_data(e_new); + void *local_event = (char *) cur_entry->event_data + cur_entry->msg.remote_event_size_bytes; + memcpy(m_new, local_event, cur_entry->msg.local_event_size_bytes); + tw_event_send(e_new); + } + s->packet_counter++; + s->vc_occupancy[0] += s->params->chunk_size; + cur_entry = return_head(s->terminal_msgs, s->terminal_msgs_tail, 0); + rc_stack_push(lp, cur_entry, delete_terminal_plus_message_list, s->st); + s->terminal_length -= s->params->chunk_size; + + cur_entry = s->terminal_msgs[0]; + + /* if there is another packet inline then schedule another send event */ + if (cur_entry != NULL && s->vc_occupancy[0] + s->params->chunk_size <= s->params->cn_vc_size) { + bf->c3 = 1; + terminal_plus_message *m_new; + ts += tw_rand_unif(lp->rng); + e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_PLUS, (void **) &m_new, NULL); + m_new->type = T_SEND; + m_new->magic = terminal_magic_num; + tw_event_send(e); + } + else { + /* If not then the LP will wait for another credit or packet generation */ + bf->c4 = 1; + s->in_send_loop = 0; + } + if (s->issueIdle) { + bf->c5 = 1; + s->issueIdle = 0; + ts += tw_rand_unif(lp->rng); + model_net_method_idle_event(ts, 0, lp); + + if (s->last_buf_full[0] > 0.0) { + bf->c6 = 1; + msg->saved_total_time = s->busy_time; + msg->saved_busy_time = s->last_buf_full[0]; + msg->saved_sample_time = s->busy_time_sample; + + s->busy_time += (tw_now(lp) - s->last_buf_full[0]); + s->busy_time_sample += (tw_now(lp) - s->last_buf_full[0]); + s->last_buf_full[0] = 0.0; + } + } + return; +} + +static void packet_arrive_rc(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + if (bf->c31) { + s->packet_fin--; + packet_fin--; + } + tw_rand_reverse_unif(lp->rng); + if (msg->path_type == MINIMAL) + minimal_count--; + // if (msg->path_type == NON_MINIMAL) + // nonmin_count--; + + N_finished_chunks--; + s->finished_chunks--; + s->fin_chunks_sample--; + + total_hops -= msg->my_N_hop; + s->total_hops -= msg->my_N_hop; + s->fin_hops_sample -= msg->my_N_hop; + dragonfly_total_time = msg->saved_total_time; + s->fin_chunks_time = msg->saved_sample_time; + s->total_time = msg->saved_avg_time; + + struct qhash_head *hash_link = NULL; + struct dfly_qhash_entry *tmp = NULL; + + struct dfly_hash_key key; + key.message_id = msg->message_id; + key.sender_id = msg->sender_lp; + + hash_link = qhash_search(s->rank_tbl, &key); + tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link); + + mn_stats *stat; + stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); + stat->recv_time = msg->saved_rcv_time; + + if (bf->c1) { + stat->recv_count--; + stat->recv_bytes -= msg->packet_size; + N_finished_packets--; + s->finished_packets--; + } + if (bf->c3) { + dragonfly_max_latency = msg->saved_available_time; + } + + if (bf->c22) { + s->max_latency = msg->saved_available_time; + } + if (bf->c7) { + // assert(!hash_link); + if (bf->c8) + tw_rand_reverse_unif(lp->rng); + N_finished_msgs--; + s->finished_msgs--; + total_msg_sz -= msg->total_size; + s->total_msg_size -= msg->total_size; + s->data_size_sample -= msg->total_size; + + struct dfly_qhash_entry *d_entry_pop = (dfly_qhash_entry *) rc_stack_pop(s->st); + qhash_add(s->rank_tbl, &key, &(d_entry_pop->hash_link)); + s->rank_tbl_pop++; + + if (s->rank_tbl_pop >= DFLY_HASH_TABLE_SIZE) + tw_error(TW_LOC, "\n Exceeded allocated qhash size, increase hash size in dragonfly model"); + + hash_link = &(d_entry_pop->hash_link); + tmp = d_entry_pop; + + if (bf->c4) + model_net_event_rc2(lp, &msg->event_rc); + } + + assert(tmp); + tmp->num_chunks--; + + if (bf->c5) { + qhash_del(hash_link); + free_tmp(tmp); + s->rank_tbl_pop--; + } + return; +} +static void send_remote_event(terminal_state *s, terminal_plus_message *msg, tw_lp *lp, tw_bf *bf, + char *event_data, int remote_event_size) +{ + void *tmp_ptr = model_net_method_get_edata(DRAGONFLY_PLUS, msg); + // tw_stime ts = g_tw_lookahead + bytes_to_ns(msg->remote_event_size_bytes, (1/s->params->cn_bandwidth)); + tw_stime ts = g_tw_lookahead + mpi_soft_overhead + tw_rand_unif(lp->rng); + if (msg->is_pull) { + bf->c4 = 1; + struct codes_mctx mc_dst = codes_mctx_set_global_direct(msg->sender_mn_lp); + struct codes_mctx mc_src = codes_mctx_set_global_direct(lp->gid); + int net_id = model_net_get_id(LP_METHOD_NM_TERM); + + model_net_set_msg_param(MN_MSG_PARAM_START_TIME, MN_MSG_PARAM_START_TIME_VAL, &(msg->msg_start_time)); + + msg->event_rc = model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category, msg->sender_lp, + msg->pull_size, ts, remote_event_size, tmp_ptr, 0, NULL, lp); + } + else { + tw_event *e = tw_event_new(msg->final_dest_gid, ts, lp); + void *m_remote = tw_event_data(e); + memcpy(m_remote, event_data, remote_event_size); + tw_event_send(e); + } + return; +} +/* packet arrives at the destination terminal */ +static void packet_arrive(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + // NIC aggregation - should this be a separate function? + // Trigger an event on receiving server + + // printf("Packet Arrived: %d hops\n",msg->my_N_hop); + + if (!s->rank_tbl) + s->rank_tbl = qhash_init(dragonfly_rank_hash_compare, dragonfly_hash_func, DFLY_HASH_TABLE_SIZE); + + struct dfly_hash_key key; + key.message_id = msg->message_id; + key.sender_id = msg->sender_lp; + + struct qhash_head *hash_link = NULL; + struct dfly_qhash_entry *tmp = NULL; + + hash_link = qhash_search(s->rank_tbl, &key); + + if (hash_link) + tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link); + + uint64_t total_chunks = msg->total_size / s->params->chunk_size; + + if (msg->total_size % s->params->chunk_size) + total_chunks++; + + if (!total_chunks) + total_chunks = 1; + + /*if(tmp) + { + if(tmp->num_chunks >= total_chunks || tmp->num_chunks < 0) + { + //tw_output(lp, "\n invalid number of chunks %d for LP %ld ", tmp->num_chunks, lp->gid); + tw_lp_suspend(lp, 0, 0); + return; + } + }*/ + assert(lp->gid == msg->dest_terminal_id); + + if (msg->packet_ID == LLU(TRACK_PKT)) + printf("\n Packet %llu arrived at lp %llu hops %d ", msg->packet_ID, LLU(lp->gid), msg->my_N_hop); + + tw_stime ts = g_tw_lookahead + s->params->credit_delay + tw_rand_unif(lp->rng); + + // no method_event here - message going to router + tw_event *buf_e; + terminal_plus_message *buf_msg; + buf_e = + model_net_method_event_new(msg->intm_lp_id, ts, lp, DRAGONFLY_PLUS_ROUTER, (void **) &buf_msg, NULL); + buf_msg->magic = router_magic_num; + buf_msg->vc_index = msg->vc_index; + buf_msg->output_chan = msg->output_chan; + buf_msg->type = R_BUFFER; + tw_event_send(buf_e); + + bf->c1 = 0; + bf->c3 = 0; + bf->c4 = 0; + bf->c7 = 0; + + /* Total overall finished chunks in simulation */ + N_finished_chunks++; + /* Finished chunks on a LP basis */ + s->finished_chunks++; + /* Finished chunks per sample */ + s->fin_chunks_sample++; + + /* WE do not allow self messages through dragonfly */ + assert(lp->gid != msg->src_terminal_id); + + uint64_t num_chunks = msg->packet_size / s->params->chunk_size; + if (msg->packet_size < s->params->chunk_size) + num_chunks++; + + if (msg->path_type == MINIMAL) + minimal_count++; + + // if (msg->path_type == NON_MINIMAL) + // nonmin_count++; + + if (msg->chunk_id == num_chunks - 1) { + bf->c31 = 1; + s->packet_fin++; + packet_fin++; + } + // if (msg->path_type != MINIMAL) + // printf("\n Wrong message path type %d ", msg->path_type); + + /* save the sample time */ + msg->saved_sample_time = s->fin_chunks_time; + s->fin_chunks_time += (tw_now(lp) - msg->travel_start_time); + + /* save the total time per LP */ + msg->saved_avg_time = s->total_time; + s->total_time += (tw_now(lp) - msg->travel_start_time); + + msg->saved_total_time = dragonfly_total_time; + dragonfly_total_time += tw_now(lp) - msg->travel_start_time; + total_hops += msg->my_N_hop; + s->total_hops += msg->my_N_hop; + s->fin_hops_sample += msg->my_N_hop; + + mn_stats *stat = model_net_find_stats(msg->category, s->dragonfly_stats_array); + msg->saved_rcv_time = stat->recv_time; + stat->recv_time += (tw_now(lp) - msg->travel_start_time); + +#if DEBUG == 1 + if (msg->packet_ID == TRACK && msg->chunk_id == num_chunks - 1 && msg->message_id == TRACK_MSG) { + printf("(%lf) [Terminal %d] packet %lld has arrived \n", tw_now(lp), (int) lp->gid, msg->packet_ID); + + printf("travel start time is %f\n", msg->travel_start_time); + + printf("My hop now is %d\n", msg->my_N_hop); + } +#endif + + /* Now retreieve the number of chunks completed from the hash and update + * them */ + void *m_data_src = model_net_method_get_edata(DRAGONFLY_PLUS, msg); + + /* If an entry does not exist then create one */ + if (!tmp) { + bf->c5 = 1; + struct dfly_qhash_entry *d_entry = (dfly_qhash_entry *) malloc(sizeof(struct dfly_qhash_entry)); + d_entry->num_chunks = 0; + d_entry->key = key; + d_entry->remote_event_data = NULL; + d_entry->remote_event_size = 0; + qhash_add(s->rank_tbl, &key, &(d_entry->hash_link)); + s->rank_tbl_pop++; + + if (s->rank_tbl_pop >= DFLY_HASH_TABLE_SIZE) + tw_error(TW_LOC, "\n Exceeded allocated qhash size, increase hash size in dragonfly model"); + + hash_link = &(d_entry->hash_link); + tmp = d_entry; + } + + assert(tmp); + tmp->num_chunks++; + + if (msg->chunk_id == num_chunks - 1) { + bf->c1 = 1; + stat->recv_count++; + stat->recv_bytes += msg->packet_size; + + N_finished_packets++; + s->finished_packets++; + } + /* if its the last chunk of the packet then handle the remote event data */ + if (msg->remote_event_size_bytes > 0 && !tmp->remote_event_data) { + /* Retreive the remote event entry */ + tmp->remote_event_data = (char *) malloc(msg->remote_event_size_bytes); + assert(tmp->remote_event_data); + tmp->remote_event_size = msg->remote_event_size_bytes; + memcpy(tmp->remote_event_data, m_data_src, msg->remote_event_size_bytes); + } + if (s->min_latency > tw_now(lp) - msg->travel_start_time) { + s->min_latency = tw_now(lp) - msg->travel_start_time; + } + if (dragonfly_max_latency < tw_now(lp) - msg->travel_start_time) { + bf->c3 = 1; + msg->saved_available_time = dragonfly_max_latency; + dragonfly_max_latency = tw_now(lp) - msg->travel_start_time; + s->max_latency = tw_now(lp) - msg->travel_start_time; + } + if (s->max_latency < tw_now(lp) - msg->travel_start_time) { + bf->c22 = 1; + msg->saved_available_time = s->max_latency; + s->max_latency = tw_now(lp) - msg->travel_start_time; + } + /* If all chunks of a message have arrived then send a remote event to the + * callee*/ + // assert(tmp->num_chunks <= total_chunks); + + if (tmp->num_chunks >= total_chunks) { + bf->c7 = 1; + + s->data_size_sample += msg->total_size; + N_finished_msgs++; + total_msg_sz += msg->total_size; + s->total_msg_size += msg->total_size; + s->finished_msgs++; + + // assert(tmp->remote_event_data && tmp->remote_event_size > 0); + if (tmp->remote_event_data && tmp->remote_event_size > 0) { + bf->c8 = 1; + send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size); + } + /* Remove the hash entry */ + qhash_del(hash_link); + rc_stack_push(lp, tmp, free_tmp, s->st); + s->rank_tbl_pop--; + } + return; +} + +void dragonfly_plus_rsample_init(router_state *s, tw_lp *lp) +{ + (void) lp; + int i = 0; + const dragonfly_plus_param *p = s->params; + + assert(p->radix); + + s->max_arr_size = MAX_STATS; + s->rsamples = (struct dfly_router_sample *) malloc(MAX_STATS * sizeof(struct dfly_router_sample)); + for (; i < s->max_arr_size; i++) { + s->rsamples[i].busy_time = (tw_stime *) malloc(sizeof(tw_stime) * p->radix); + s->rsamples[i].link_traffic_sample = (int64_t *) malloc(sizeof(int64_t) * p->radix); + } +} +void dragonfly_plus_rsample_rc_fn(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + (void) bf; + (void) lp; + (void) msg; + + s->op_arr_size--; + int cur_indx = s->op_arr_size; + struct dfly_router_sample stat = s->rsamples[cur_indx]; + + const dragonfly_plus_param *p = s->params; + int i = 0; + + for (; i < p->radix; i++) { + s->busy_time_sample[i] = stat.busy_time[i]; + s->link_traffic_sample[i] = stat.link_traffic_sample[i]; + } + + for (i = 0; i < p->radix; i++) { + stat.busy_time[i] = 0; + stat.link_traffic_sample[i] = 0; + } + s->fwd_events = stat.fwd_events; + s->rev_events = stat.rev_events; +} + +void dragonfly_plus_rsample_fn(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + (void) bf; + (void) lp; + (void) msg; + + const dragonfly_plus_param *p = s->params; + + if (s->op_arr_size >= s->max_arr_size) { + struct dfly_router_sample *tmp = + (dfly_router_sample *) malloc((MAX_STATS + s->max_arr_size) * sizeof(struct dfly_router_sample)); + memcpy(tmp, s->rsamples, s->op_arr_size * sizeof(struct dfly_router_sample)); + free(s->rsamples); + s->rsamples = tmp; + s->max_arr_size += MAX_STATS; + } + + int i = 0; + int cur_indx = s->op_arr_size; + + s->rsamples[cur_indx].router_id = s->router_id; + s->rsamples[cur_indx].end_time = tw_now(lp); + s->rsamples[cur_indx].fwd_events = s->fwd_events; + s->rsamples[cur_indx].rev_events = s->rev_events; + + for (; i < p->radix; i++) { + s->rsamples[cur_indx].busy_time[i] = s->busy_time_sample[i]; + s->rsamples[cur_indx].link_traffic_sample[i] = s->link_traffic_sample[i]; + } + + s->op_arr_size++; + + /* clear up the current router stats */ + s->fwd_events = 0; + s->rev_events = 0; + + for (i = 0; i < p->radix; i++) { + s->busy_time_sample[i] = 0; + s->link_traffic_sample[i] = 0; + } +} + +void dragonfly_plus_rsample_fin(router_state *s, tw_lp *lp) +{ + (void) lp; + const dragonfly_plus_param *p = s->params; + + if (s->router_id == 0) { + /* write metadata file */ + char meta_fname[64]; + sprintf(meta_fname, "dragonfly-plus-router-sampling.meta"); + + FILE *fp = fopen(meta_fname, "w"); + fprintf(fp, + "Router sample struct format: \nrouter_id (tw_lpid) \nbusy time for each of the %d links " + "(double) \n" + "link traffic for each of the %d links (int64_t) \nsample end time (double) forward events " + "per sample \nreverse events per sample ", + p->radix, p->radix); + fclose(fp); + } + char rt_fn[MAX_NAME_LENGTH]; + if (strcmp(router_sample_file, "") == 0) + sprintf(rt_fn, "dragonfly-plus-router-sampling-%ld.bin", g_tw_mynode); + else + sprintf(rt_fn, "%s-%ld.bin", router_sample_file, g_tw_mynode); + + int i = 0; + + int size_sample = sizeof(tw_lpid) + p->radix * (sizeof(int64_t) + sizeof(tw_stime)) + + sizeof(tw_stime) + 2 * sizeof(long); + FILE *fp = fopen(rt_fn, "a"); + fseek(fp, sample_rtr_bytes_written, SEEK_SET); + + for (; i < s->op_arr_size; i++) { + fwrite((void *) &(s->rsamples[i].router_id), sizeof(tw_lpid), 1, fp); + fwrite(s->rsamples[i].busy_time, sizeof(tw_stime), p->radix, fp); + fwrite(s->rsamples[i].link_traffic_sample, sizeof(int64_t), p->radix, fp); + fwrite((void *) &(s->rsamples[i].end_time), sizeof(tw_stime), 1, fp); + fwrite((void *) &(s->rsamples[i].fwd_events), sizeof(long), 1, fp); + fwrite((void *) &(s->rsamples[i].rev_events), sizeof(long), 1, fp); + } + sample_rtr_bytes_written += (s->op_arr_size * size_sample); + fclose(fp); +} +void dragonfly_plus_sample_init(terminal_state *s, tw_lp *lp) +{ + (void) lp; + s->fin_chunks_sample = 0; + s->data_size_sample = 0; + s->fin_hops_sample = 0; + s->fin_chunks_time = 0; + s->busy_time_sample = 0; + + s->op_arr_size = 0; + s->max_arr_size = MAX_STATS; + + s->sample_stat = (dfly_cn_sample *) malloc(MAX_STATS * sizeof(struct dfly_cn_sample)); +} +void dragonfly_plus_sample_rc_fn(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + (void) lp; + (void) bf; + (void) msg; + + s->op_arr_size--; + int cur_indx = s->op_arr_size; + struct dfly_cn_sample stat = s->sample_stat[cur_indx]; + s->busy_time_sample = stat.busy_time_sample; + s->fin_chunks_time = stat.fin_chunks_time; + s->fin_hops_sample = stat.fin_hops_sample; + s->data_size_sample = stat.data_size_sample; + s->fin_chunks_sample = stat.fin_chunks_sample; + s->fwd_events = stat.fwd_events; + s->rev_events = stat.rev_events; + + stat.busy_time_sample = 0; + stat.fin_chunks_time = 0; + stat.fin_hops_sample = 0; + stat.data_size_sample = 0; + stat.fin_chunks_sample = 0; + stat.end_time = 0; + stat.terminal_id = 0; + stat.fwd_events = 0; + stat.rev_events = 0; +} + +void dragonfly_plus_sample_fn(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + (void) lp; + (void) msg; + (void) bf; + + if (s->op_arr_size >= s->max_arr_size) { + /* In the worst case, copy array to a new memory location, its very + * expensive operation though */ + struct dfly_cn_sample *tmp = + (dfly_cn_sample *) malloc((MAX_STATS + s->max_arr_size) * sizeof(struct dfly_cn_sample)); + memcpy(tmp, s->sample_stat, s->op_arr_size * sizeof(struct dfly_cn_sample)); + free(s->sample_stat); + s->sample_stat = tmp; + s->max_arr_size += MAX_STATS; + } + + int cur_indx = s->op_arr_size; + + s->sample_stat[cur_indx].terminal_id = s->terminal_id; + s->sample_stat[cur_indx].fin_chunks_sample = s->fin_chunks_sample; + s->sample_stat[cur_indx].data_size_sample = s->data_size_sample; + s->sample_stat[cur_indx].fin_hops_sample = s->fin_hops_sample; + s->sample_stat[cur_indx].fin_chunks_time = s->fin_chunks_time; + s->sample_stat[cur_indx].busy_time_sample = s->busy_time_sample; + s->sample_stat[cur_indx].end_time = tw_now(lp); + s->sample_stat[cur_indx].fwd_events = s->fwd_events; + s->sample_stat[cur_indx].rev_events = s->rev_events; + + s->op_arr_size++; + s->fin_chunks_sample = 0; + s->data_size_sample = 0; + s->fin_hops_sample = 0; + s->fwd_events = 0; + s->rev_events = 0; + s->fin_chunks_time = 0; + s->busy_time_sample = 0; +} + +void dragonfly_plus_sample_fin(terminal_state *s, tw_lp *lp) +{ + (void) lp; + + + if (!g_tw_mynode) { + /* write metadata file */ + char meta_fname[64]; + sprintf(meta_fname, "dragonfly-cn-sampling.meta"); + + FILE *fp = fopen(meta_fname, "w"); + fprintf( + fp, + "Compute node sample format\nterminal_id (tw_lpid) \nfinished chunks (long)" + "\ndata size per sample (long) \nfinished hops (double) \ntime to finish chunks (double)" + "\nbusy time (double)\nsample end time(double) \nforward events (long) \nreverse events (long)"); + fclose(fp); + } + + char rt_fn[MAX_NAME_LENGTH]; + if (strncmp(cn_sample_file, "", 10) == 0) + sprintf(rt_fn, "dragonfly-cn-sampling-%ld.bin", g_tw_mynode); + else + sprintf(rt_fn, "%s-%ld.bin", cn_sample_file, g_tw_mynode); + + FILE *fp = fopen(rt_fn, "a"); + fseek(fp, sample_bytes_written, SEEK_SET); + fwrite(s->sample_stat, sizeof(struct dfly_cn_sample), s->op_arr_size, fp); + fclose(fp); + + sample_bytes_written += (s->op_arr_size * sizeof(struct dfly_cn_sample)); +} + +static void terminal_buf_update_rc(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + s->vc_occupancy[0] += s->params->chunk_size; + codes_local_latency_reverse(lp); + if (bf->c1) { + s->in_send_loop = 0; + } + + return; +} +/* update the compute node-router channel buffer */ +static void terminal_buf_update(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + bf->c1 = 0; + bf->c2 = 0; + bf->c3 = 0; + + tw_stime ts = codes_local_latency(lp); + s->vc_occupancy[0] -= s->params->chunk_size; + + if (s->in_send_loop == 0 && s->terminal_msgs[0] != NULL) { + terminal_plus_message *m; + bf->c1 = 1; + tw_event *e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_PLUS, (void **) &m, NULL); + m->type = T_SEND; + m->magic = terminal_magic_num; + s->in_send_loop = 1; + tw_event_send(e); + } + return; +} + +void terminal_plus_event(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + s->fwd_events++; + //*(int *)bf = (int)0; + assert(msg->magic == terminal_magic_num); + + rc_stack_gc(lp, s->st); + switch (msg->type) { + case T_GENERATE: + packet_generate(s, bf, msg, lp); + break; + + case T_ARRIVE: + packet_arrive(s, bf, msg, lp); + break; + + case T_SEND: + packet_send(s, bf, msg, lp); + break; + + case T_BUFFER: + terminal_buf_update(s, bf, msg, lp); + break; + + default: + printf("\n LP %d Terminal message type not supported %d ", (int) lp->gid, msg->type); + tw_error(TW_LOC, "Msg type not supported"); + } +} + +void dragonfly_plus_terminal_final(terminal_state *s, tw_lp *lp) +{ + model_net_print_stats(lp->gid, s->dragonfly_stats_array); + + if (s->terminal_id == 0) { + char meta_filename[64]; + sprintf(meta_filename, "dragonfly-msg-stats.meta"); + + FILE *fp = fopen(meta_filename, "w+"); + fprintf(fp, + "# Format <# Flits/Packets " + "finished> \n"); + } + int written = 0; + + written += sprintf(s->output_buf + written, "%llu %u %llu %lf %ld %lf %lf %lf %lf\n", LLU(lp->gid), + s->terminal_id, LLU(s->total_msg_size), s->total_time / s->finished_chunks, + s->finished_packets, (double) s->total_hops / s->finished_chunks, s->busy_time, + s->max_latency, s->min_latency); + + lp_io_write(lp->gid, (char *) "dragonfly-msg-stats", written, s->output_buf); + + if (s->terminal_msgs[0] != NULL) + printf("[%llu] leftover terminal messages \n", LLU(lp->gid)); + + + // if(s->packet_gen != s->packet_fin) + // printf("\n generated %d finished %d ", s->packet_gen, s->packet_fin); + + if (s->rank_tbl) + qhash_finalize(s->rank_tbl); + + rc_stack_destroy(s->st); + free(s->vc_occupancy); + free(s->terminal_msgs); + free(s->terminal_msgs_tail); +} + +void dragonfly_plus_router_final(router_state *s, tw_lp *lp) +{ + free(s->global_channel); + int i, j; + for (i = 0; i < s->params->radix; i++) { + for (j = 0; j < s->params->num_vcs; j++) { + if (s->queued_msgs[i][j] != NULL) { + printf("[%llu] leftover queued messages %d %d %d\n", LLU(lp->gid), i, j, + s->vc_occupancy[i][j]); + } + if (s->pending_msgs[i][j] != NULL) { + printf("[%llu] lefover pending messages %d %d\n", LLU(lp->gid), i, j); + } + } + } + + rc_stack_destroy(s->st); + + /*MM: These statistics will need to be updated for dragonfly plus. + * Especially the meta file information on router ports still have green + * and black links. */ + const dragonfly_plus_param *p = s->params; + int written = 0; + if (!s->router_id) { + written = sprintf(s->output_buf, "DRAGONFLY-PLUS STATS OUTPUT NOT IMPLEMENTED\n"); + written = + sprintf(s->output_buf, "# Format "); + } + written += sprintf(s->output_buf + written, "\n %llu %d %d", LLU(lp->gid), s->router_id / p->num_routers, + s->router_id % p->num_routers); + for (int d = 0; d < p->radix; d++) + written += sprintf(s->output_buf + written, " %lf", s->busy_time[d]); + + sprintf(s->output_buf + written, "\n"); + lp_io_write(lp->gid, (char *) "dragonfly-plus-router-stats", written, s->output_buf); + + written = 0; + if (!s->router_id) { + written = sprintf(s->output_buf2, "DRAGONFLY-PLUS TRAFFIC OUTPUT NOT IMPLEMENTED\n"); + written = + sprintf(s->output_buf2, "# Format "); + } + written += sprintf(s->output_buf2 + written, "\n %llu %d %d", LLU(lp->gid), s->router_id / p->num_routers, + s->router_id % p->num_routers); + + for (int d = 0; d < p->radix; d++) + written += sprintf(s->output_buf2 + written, " %lld", LLD(s->link_traffic[d])); + + lp_io_write(lp->gid, (char *) "dragonfly-plus-router-traffic", written, s->output_buf2); +} + +//Returns a vector of possible next stops that follow a minimal route to the destination +static vector< Connection > get_possible_minimal_next_stops(router_state *s, + tw_bf *bf, + terminal_plus_message *msg, + tw_lp *lp, + int fdest_router_id) +{ + int local_router_id = s->router_id; + int my_group_id = s->router_id / s->params->num_routers; + int fdest_group_id = fdest_router_id / s->params->num_routers; + int origin_grp_id = msg->origin_router_id / s->params->num_routers; + + vector< Connection > possible_next_conns = vector< Connection >(); + + if (fdest_router_id == local_router_id) { + assert(s->dfp_router_type == LEAF); + int term_id = msg->dfp_dest_terminal_id; + possible_next_conns = s->connMan->get_connections_to_gid(term_id, CONN_TERMINAL); + } + else if (s->group_id == fdest_group_id) { //we need to forward to the assigned router for the destination terminal + if (s->dfp_router_type == LEAF) { + //we need to send to a spine who can send it back down to the dest leaf in our group + possible_next_conns = s->connMan->get_connections_by_type(CONN_LOCAL); + } + else { + assert(s->dfp_router_type == SPINE); + int assigned_router = dragonfly_plus_get_assigned_router_id(msg->dfp_dest_terminal_id, s->params); + possible_next_conns = s->connMan->get_connections_to_gid(assigned_router, CONN_LOCAL); + } + } + else { + //Then packet direction is upward to another group + if (s->dfp_router_type == LEAF) { + //we need to get a list of connections to spine routers in our group that can connect to the fdest group + + set poss_router_id_set; + for(int i = 0; i < connectionList[my_group_id][fdest_group_id].size(); i++) + { + int poss_router_id = connectionList[my_group_id][fdest_group_id][i]; + if (poss_router_id_set.count(poss_router_id) == 0) { //if we haven't added the connections from poss_router_id yet + vector< Connection > conns = s->connMan->get_connections_to_gid(poss_router_id, CONN_LOCAL); + possible_next_conns.insert(possible_next_conns.end(), conns.begin(), conns.end()); + poss_router_id_set.insert(poss_router_id); + } + } + } + else { + assert(s->dfp_router_type == SPINE); + possible_next_conns = s->connMan->get_connections_to_group(fdest_group_id); //gets connections to fdest group + } + } + return possible_next_conns; +} + +//Returns a vector of possible next stops that follow a nonminimal route - +static vector< Connection > get_possible_nonminimal_next_stops(router_state *s, + tw_bf *bf, + terminal_plus_message *msg, + tw_lp *lp, + int fdest_router_id) +{ + int local_router_id = s->router_id; + int my_group_id = s->router_id / s->params->num_routers; + int fdest_group_id = fdest_router_id / s->params->num_routers; + int origin_grp_id = msg->origin_router_id / s->params->num_routers; + + vector< Connection > possible_next_conns; + + if (s->group_id == fdest_group_id) { + return possible_next_conns; //empty because there is only minimal connections within the destination group + } + else { + if(s->dfp_router_type == LEAF) { + possible_next_conns = s->connMan->get_connections_by_type(CONN_LOCAL); //I am a leaf, my local connections are all to spine routers in my group + } + else { //SPINE + assert(s->dfp_router_type == SPINE); + bool in_intermediate_group = (my_group_id != origin_grp_id) && (my_group_id != fdest_group_id); + + if (in_intermediate_group) { //then the nonminimal paths are those that go to leafs in my group + possible_next_conns = s->connMan->get_connections_by_type(CONN_LOCAL); + } + else { // Then we have to be in source group - nonminimal paths are those that go to any other group other than destination + + for(int i = 0; i < s->params->num_groups; i++) + { + if (i != fdest_group_id) { + vector< Connection > conns_to_group = s->connMan->get_connections_to_group(i); + possible_next_conns.insert(possible_next_conns.end(),conns_to_group.begin(),conns_to_group.end()); + } + } + } + } + } + return possible_next_conns; +} + +static int get_min_hops_to_dest_from_conn(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, Connection conn) +{ + int my_type = s->dfp_router_type; + int next_hops_type = dragonfly_plus_get_router_type(conn.dest_gid, s->params); + + int dfp_dest_terminal_id = msg->dfp_dest_terminal_id; + int fdest_router_id = dragonfly_plus_get_assigned_router_id(dfp_dest_terminal_id, s->params); + int fdest_group_id = fdest_router_id / s->params->num_routers; + + if (msg->dfp_upward_channel_flag) + { + if (my_type == SPINE) + return 2; //Next Spine -> Leaf -> dest_term + else + return 3; //Next Spine -> Spine -> Leaf -> dest_term + } + + if(conn.dest_group_id == fdest_group_id) { + if (next_hops_type == SPINE) + return 2; //Next Spine -> Leaf -> dest_term + else { + assert(next_hops_type == LEAF); + return 1; //Next Leaf -> dest_term + } + } + else { //next is not in final destination group + if (next_hops_type == SPINE) + return 3; //Next Spine -> Spine -> Leaf -> dest_term + else { + assert(next_hops_type == LEAF); + return 4; //Next Leaf -> Spine -> Spine -> Leaf -> dest_term + } + } +} + +/** + * Scores a connection based on the metric provided in the function, lower score is better + */ +static int dfp_score_connection(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, Connection conn) +{ + //Currently just looking at VC Occupancy and message queue length like dragonfly custom + int score = 0; + int port = conn.port; + + switch(scoring) { + case ALPHA: + { + for(int k=0; k < s->params->num_vcs; k++) + { + score += s->vc_occupancy[port][k]; + } + score += s->queued_count[port]; + break; + } + case BETA: + { + int base_score = 0; + for(int k=0; k < s->params->num_vcs; k++) + { + base_score += s->vc_occupancy[port][k]; + } + base_score += s->queued_count[port]; + score = base_score * get_min_hops_to_dest_from_conn(s, bf, msg, lp, conn); + break; + } + } + return score; +} + +static vector< Connection > dfp_select_two_connections(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp, vector< Connection > conns) +{ + if(conns.size() < 2) { + tw_rand_integer(lp->rng,0,2); + tw_rand_integer(lp->rng,0,2); + if(conns.size() == 1) + return conns; + if(conns.size() == 0) + return vector< Connection>(); + } + + + int rand_sel_1, rand_sel_2; + vector indices; + vector::iterator it; + + for(int i = 0; i < conns.size(); i++) + { + indices.push_back(i); + } + + // printf("Selecting 1st from %d\n",indices.size()); + int pi = tw_rand_integer(lp->rng, 0, indices.size()-1); + rand_sel_1 = indices[pi]; + for(it = indices.begin(); it != indices.end(); it++) //because std::find wasn't working :/ + { + if (*it == rand_sel_1) + break; + } + indices.erase(it); //erases the number located at pos pi + + // printf("Selecting 2nd from %d\n",indices.size()); + pi = tw_rand_integer(lp->rng, 0, indices.size()-1); + rand_sel_2 = indices[pi]; + + for(it = indices.begin(); it != indices.end(); it++) //because std::find wasn't working :/ + { + if (*it == rand_sel_2) + break; + } + indices.erase(it); //erases the number located at pos pi + + vector< Connection > selected_conns; + selected_conns.push_back(conns[rand_sel_1]); + selected_conns.push_back(conns[rand_sel_2]); + return selected_conns; +} + +static Connection do_dfp_routing(router_state *s, + tw_bf *bf, + terminal_plus_message *msg, + tw_lp *lp, + int fdest_router_id) +{ + + int local_router_id = s->router_id; + int my_group_id = s->router_id / s->params->num_routers; + int fdest_group_id = fdest_router_id / s->params->num_routers; + int origin_grp_id = msg->origin_router_id / s->params->num_routers; + bool in_intermediate_group = (my_group_id != origin_grp_id) && (my_group_id != fdest_group_id); + + + vector< Connection > poss_min_next_stops = get_possible_minimal_next_stops(s, bf, msg, lp, fdest_router_id); + vector< Connection > poss_non_min_next_stops = get_possible_nonminimal_next_stops(s, bf, msg, lp, fdest_router_id); + + Connection theConn; + + if (isRoutingAdaptive(routing)) { + bool choose_minimal = false; + bool turning_upward = false; + + int num_min_poss = poss_min_next_stops.size(); + int num_nonmin_poss = poss_non_min_next_stops.size(); + + if (msg->dfp_upward_channel_flag == 1 || (my_group_id == fdest_group_id)) + choose_minimal = true; //in both of these cases minimal should always be chosen + + + vector< Connection > two_minimal_conns = dfp_select_two_connections(s, bf, msg, lp, poss_min_next_stops); + vector< Connection > two_nonminimal_conns = dfp_select_two_connections(s, bf, msg, lp, poss_non_min_next_stops); + int num_conns = two_minimal_conns.size() + two_nonminimal_conns.size(); //we selected two from each //TODO maybe make this configurable + + + //calculate the best connections for minimal and nonminimal + int scores[num_conns]; + for(int i = 0; i < num_conns; i++) + { + if(i < (two_minimal_conns.size()) ) + scores[i] = dfp_score_connection(s, bf, msg, lp, two_minimal_conns[i]); + else + scores[i] = dfp_score_connection(s, bf, msg, lp, two_nonminimal_conns[i-two_minimal_conns.size()]); + } + + int best_min_score = INT_MAX; + int best_min_score_index = 0; + for(int i = 0; i < two_minimal_conns.size(); i++) + { + if (scores[i] < best_min_score) { + best_min_score = scores[i]; + best_min_score_index = i; + } + } + + int best_non_min_score = INT_MAX; + int best_non_min_score_index = 0; + for(int i = two_minimal_conns.size(); i < num_conns; i++) + { + if (scores[i] < best_non_min_score) { + best_non_min_score = scores[i]; + best_non_min_score_index = i-two_minimal_conns.size(); + } + } + //end score calculation + + if (best_min_score <= best_non_min_score) { //ties go to minimal + choose_minimal = true; + } + + //intermediate groups have restrictions on how they have to route to avoid livelock + if (in_intermediate_group) { + if (s->dfp_router_type == SPINE) { //intermediate spine + if (choose_minimal) + msg->dfp_upward_channel_flag = 1; //only if it chose minimal should this be flipped on + } + else { + assert(s->dfp_router_type == LEAF); //intermediate leaf always sends + choose_minimal = true; + msg->dfp_upward_channel_flag = 1; + } + } + + if (choose_minimal) { + theConn = two_minimal_conns[best_min_score_index]; + } + else { + theConn = two_nonminimal_conns[best_non_min_score_index]; + } + } + + else { //routing algorithm is specified in msg->path_type + bool choose_minimal = false; + if ( msg->path_type == MINIMAL || msg->dfp_upward_channel_flag == 1 || (my_group_id == fdest_group_id) ) + choose_minimal = true; + else { + if (in_intermediate_group) { + if (s->dfp_router_type == SPINE) { //INTERMEDIATE SPINE + if (msg->path_type == NON_MINIMAL_SPINE) { + //we then follow minimal + choose_minimal = true; + msg->dfp_upward_channel_flag = 1; + } + else { + assert(msg->path_type == NON_MINIMAL_LEAF); + choose_minimal = false; + } + } + else { //INTERMEDIATE LEAF + choose_minimal = true; + msg->dfp_upward_channel_flag = 1; + } + } + else { //must be in source group + assert(my_group_id == origin_grp_id); + choose_minimal = false; + } + } + + if (choose_minimal == true) { + assert(poss_min_next_stops.size() > 0); + int rand_sel = tw_rand_integer(lp->rng, 0, poss_min_next_stops.size()-1); + theConn = poss_min_next_stops[rand_sel]; + } + else { + assert(poss_non_min_next_stops.size() > 0); + int rand_sel = tw_rand_integer(lp->rng, 0, poss_non_min_next_stops.size()-1); + theConn = poss_non_min_next_stops[rand_sel]; + } + } + + return theConn; +} + +static void router_packet_receive_rc(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + router_rev_ecount++; + router_ecount--; + + int output_port = msg->saved_vc; + int output_chan = msg->saved_channel; + + if( isRoutingAdaptive(routing) ) { + //we need to reverse 4 + for(int i =0; i < 4; i++) + tw_rand_reverse_unif(lp->rng); + } + else { + tw_rand_reverse_unif(lp->rng); + } + + tw_rand_reverse_unif(lp->rng); + if (bf->c2) { + tw_rand_reverse_unif(lp->rng); + terminal_plus_message_list *tail = + return_tail(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], output_chan); + delete_terminal_plus_message_list(tail); + s->vc_occupancy[output_port][output_chan] -= s->params->chunk_size; + if (bf->c3) { + codes_local_latency_reverse(lp); + s->in_send_loop[output_port] = 0; + } + } + if (bf->c4) { + if (bf->c22) { + s->last_buf_full[output_port][output_chan] = msg->saved_busy_time; + } + delete_terminal_plus_message_list( + return_tail(s->queued_msgs[output_port], s->queued_msgs_tail[output_port], output_chan)); + s->queued_count[output_port] -= s->params->chunk_size; + } +} + +/* MM: This will need changes for the port selection and routing part. The + * progressive adaptive routing would need modification as well. */ +/* Packet arrives at the router and a credit is sent back to the sending terminal/router */ +static void router_packet_receive(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + router_ecount++; + + tw_stime ts; + + int next_stop = -1, output_port = -1, output_chan = -1, adap_chan = -1; + int dfp_dest_terminal_id = msg->dfp_dest_terminal_id; + int dest_router_id = dragonfly_plus_get_assigned_router_id(dfp_dest_terminal_id, s->params); + // int dest_router_id = codes_mapping_get_lp_relative_id(msg->dest_terminal_id, 0, 0) / s->params->num_cn; + int local_grp_id = s->router_id / s->params->num_routers; + int src_grp_id = msg->origin_router_id / s->params->num_routers; + int dest_grp_id = dest_router_id / s->params->num_routers; + int intm_router_id; + short prev_path_type = 0, next_path_type = 0; + + terminal_plus_message_list *cur_chunk = + (terminal_plus_message_list *) malloc(sizeof(terminal_plus_message_list)); + init_terminal_plus_message_list(cur_chunk, msg); + + cur_chunk->msg.path_type = MINIMAL; + if ( !isRoutingMinimal(routing) ) + { + if (!cur_chunk->msg.dfp_upward_channel_flag) + { + cur_chunk->msg.path_type = routing; //if we're on an upward channel then we follow minimal routing + } + } + + Connection next_stop_conn = do_dfp_routing(s, bf, &(cur_chunk->msg), lp, dest_router_id); + + output_port = next_stop_conn.port; + + if (next_stop_conn.conn_type != CONN_TERMINAL) { + int id = next_stop_conn.dest_gid; + tw_lpid router_dest_id; + codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, id / num_routers_per_mgrp, id % num_routers_per_mgrp, &router_dest_id); + next_stop = router_dest_id; + } + else { + next_stop = cur_chunk->msg.dest_terminal_id; + } + + + //From here the output port is known and output_chan is determined shortly + assert(output_port >= 0); + cur_chunk->msg.vc_index = output_port; + cur_chunk->msg.next_stop = next_stop; + + int max_vc_size = s->params->cn_vc_size; + + output_chan = 0; + + if(cur_chunk->msg.path_type == MINIMAL) + { + output_chan = 0; + } + if(cur_chunk->msg.dfp_upward_channel_flag) + { + output_chan = 1; + } + + cur_chunk->msg.output_chan = output_chan; + cur_chunk->msg.my_N_hop++; + + if (output_port >= s->params->radix) + tw_error(TW_LOC, "\n%d: Output port greater than router radix %d ",s->router_id, output_port); + + if (output_chan >= s->params->num_vcs || output_chan < 0) + printf( + "\n Packet %llu Output chan %d output port %d my rid %d dest rid %d path %d my gid %d dest gid " + "%d", + cur_chunk->msg.packet_ID, output_chan, output_port, s->router_id, dest_router_id, + cur_chunk->msg.path_type, src_grp_id, dest_grp_id); + + assert(output_chan < s->params->num_vcs && output_chan >= 0); + + if (msg->remote_event_size_bytes > 0) { + void *m_data_src = model_net_method_get_edata(DRAGONFLY_PLUS_ROUTER, msg); + cur_chunk->event_data = (char *) malloc(msg->remote_event_size_bytes); + memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes); + } + + if (s->vc_occupancy[output_port][output_chan] + s->params->chunk_size <= max_vc_size) { + bf->c2 = 1; + router_credit_send(s, msg, lp, -1); + append_to_terminal_plus_message_list(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], + output_chan, cur_chunk); + s->vc_occupancy[output_port][output_chan] += s->params->chunk_size; + if (s->in_send_loop[output_port] == 0) { + bf->c3 = 1; + terminal_plus_message *m; + ts = codes_local_latency(lp); + tw_event *e = + model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_PLUS_ROUTER, (void **) &m, NULL); + m->type = R_SEND; + m->magic = router_magic_num; + m->vc_index = output_port; + m->dfp_upward_channel_flag = msg->dfp_upward_channel_flag; + + tw_event_send(e); + s->in_send_loop[output_port] = 1; + } + } + else { + bf->c4 = 1; + cur_chunk->msg.saved_vc = msg->vc_index; + cur_chunk->msg.saved_channel = msg->output_chan; + append_to_terminal_plus_message_list(s->queued_msgs[output_port], s->queued_msgs_tail[output_port], + output_chan, cur_chunk); + s->queued_count[output_port] += s->params->chunk_size; + + /* a check for pending msgs is non-empty then we dont set anything. If + * that is empty then we check if last_buf_full is set or not. If already + * set then we don't overwrite it. If two packets arrive next to each other + * then the first person should be setting it. */ + if (s->pending_msgs[output_port][output_chan] == NULL && + s->last_buf_full[output_port][output_chan] == 0.0) { + bf->c22 = 1; + msg->saved_busy_time = s->last_buf_full[output_port][output_chan]; + s->last_buf_full[output_port][output_chan] = tw_now(lp); + } + } + + msg->saved_vc = output_port; + msg->saved_channel = output_chan; + return; +} + +static void router_packet_send_rc(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + router_ecount--; + router_rev_ecount++; + + int output_port = msg->saved_vc; + int output_chan = msg->saved_channel; + if (bf->c1) { + s->in_send_loop[output_port] = 1; + return; + } + + tw_rand_reverse_unif(lp->rng); + + terminal_plus_message_list *cur_entry = (terminal_plus_message_list *) rc_stack_pop(s->st); + assert(cur_entry); + + if (bf->c11) { + s->link_traffic[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size; + s->link_traffic_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size; + } + if (bf->c12) { + s->link_traffic[output_port] -= s->params->chunk_size; + s->link_traffic_sample[output_port] -= s->params->chunk_size; + } + s->next_output_available_time[output_port] = msg->saved_available_time; + + prepend_to_terminal_plus_message_list(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], + output_chan, cur_entry); + + if (bf->c3) { + tw_rand_reverse_unif(lp->rng); + } + + if (bf->c4) { + s->in_send_loop[output_port] = 1; + } +} + +/* MM: I think this mostly stays the same. */ +/* routes the current packet to the next stop */ +static void router_packet_send(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + router_ecount++; + + tw_stime ts; + tw_event *e; + terminal_plus_message *m; + int output_port = msg->vc_index; + + terminal_plus_message_list *cur_entry = NULL; + + int output_chan = s->params->num_vcs - 1; + for (int k = s->params->num_vcs - 1; k >= 0; k--) { + cur_entry = s->pending_msgs[output_port][k]; + if (cur_entry != NULL) { + output_chan = k; + break; + } + } + + msg->saved_vc = output_port; + msg->saved_channel = output_chan; + + if (cur_entry == NULL) { + bf->c1 = 1; + s->in_send_loop[output_port] = 0; + return; + } + + int to_terminal = 1, global = 0; + double delay = s->params->cn_delay; + double bandwidth = s->params->cn_bandwidth; + + if (output_port < s->params->intra_grp_radix) { + to_terminal = 0; + delay = s->params->local_delay; + bandwidth = s->params->local_bandwidth; + } + else if (output_port < s->params->intra_grp_radix + s->params->num_global_connections) { + to_terminal = 0; + global = 1; + delay = s->params->global_delay; + bandwidth = s->params->global_bandwidth; + } + + uint64_t num_chunks = cur_entry->msg.packet_size / s->params->chunk_size; + if (msg->packet_size < s->params->chunk_size) + num_chunks++; + + double bytetime = delay; + + if (cur_entry->msg.packet_size == 0) + bytetime = bytes_to_ns(CREDIT_SIZE, bandwidth); + + if ((cur_entry->msg.packet_size < s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1)) + bytetime = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, bandwidth); + + ts = g_tw_lookahead + tw_rand_unif(lp->rng) + bytetime + s->params->router_delay; + + msg->saved_available_time = s->next_output_available_time[output_port]; + s->next_output_available_time[output_port] = maxd(s->next_output_available_time[output_port], tw_now(lp)); + s->next_output_available_time[output_port] += ts; + + ts = s->next_output_available_time[output_port] - tw_now(lp); + // dest can be a router or a terminal, so we must check + void *m_data; + if (to_terminal) { + assert(cur_entry->msg.next_stop == cur_entry->msg.dest_terminal_id); + e = model_net_method_event_new(cur_entry->msg.next_stop, + s->next_output_available_time[output_port] - tw_now(lp), lp, + DRAGONFLY_PLUS, (void **) &m, &m_data); + } + else { + e = model_net_method_event_new(cur_entry->msg.next_stop, + s->next_output_available_time[output_port] - tw_now(lp), lp, + DRAGONFLY_PLUS_ROUTER, (void **) &m, &m_data); + } + memcpy(m, &cur_entry->msg, sizeof(terminal_plus_message)); + if (m->remote_event_size_bytes) { + memcpy(m_data, cur_entry->event_data, m->remote_event_size_bytes); + } + + if (global) + m->last_hop = GLOBAL; + else + m->last_hop = LOCAL; + + m->intm_lp_id = lp->gid; + m->magic = router_magic_num; + + if ((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1)) { + bf->c11 = 1; + s->link_traffic[output_port] += (cur_entry->msg.packet_size % s->params->chunk_size); + s->link_traffic_sample[output_port] += (cur_entry->msg.packet_size % s->params->chunk_size); + } + else { + bf->c12 = 1; + s->link_traffic[output_port] += s->params->chunk_size; + s->link_traffic_sample[output_port] += s->params->chunk_size; + } + + /* Determine the event type. If the packet has arrived at the final + * destination router then it should arrive at the destination terminal + * next.*/ + if (to_terminal) { + m->type = T_ARRIVE; + m->magic = terminal_magic_num; + } + else { + /* The packet has to be sent to another router */ + m->magic = router_magic_num; + m->type = R_ARRIVE; + } + tw_event_send(e); + + cur_entry = return_head(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], output_chan); + rc_stack_push(lp, cur_entry, delete_terminal_plus_message_list, s->st); + + s->next_output_available_time[output_port] -= s->params->router_delay; + ts -= s->params->router_delay; + + cur_entry = NULL; + + for (int k = s->params->num_vcs - 1; k >= 0; k--) { + cur_entry = s->pending_msgs[output_port][k]; + if (cur_entry != NULL) + break; + } + if (cur_entry != NULL) { + bf->c3 = 1; + terminal_plus_message *m_new; + ts += g_tw_lookahead + tw_rand_unif(lp->rng); + e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_PLUS_ROUTER, (void **) &m_new, NULL); + m_new->type = R_SEND; + m_new->magic = router_magic_num; + m_new->vc_index = output_port; + tw_event_send(e); + } + else { + bf->c4 = 1; + s->in_send_loop[output_port] = 0; + } + return; +} + +static void router_buf_update_rc(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + int indx = msg->vc_index; + int output_chan = msg->output_chan; + s->vc_occupancy[indx][output_chan] += s->params->chunk_size; + if (bf->c3) { + s->busy_time[indx] = msg->saved_rcv_time; + s->busy_time_sample[indx] = msg->saved_sample_time; + s->last_buf_full[indx][output_chan] = msg->saved_busy_time; + } + if (bf->c1) { + terminal_plus_message_list *head = + return_tail(s->pending_msgs[indx], s->pending_msgs_tail[indx], output_chan); + tw_rand_reverse_unif(lp->rng); + prepend_to_terminal_plus_message_list(s->queued_msgs[indx], s->queued_msgs_tail[indx], output_chan, + head); + s->vc_occupancy[indx][output_chan] -= s->params->chunk_size; + s->queued_count[indx] += s->params->chunk_size; + } + if (bf->c2) { + codes_local_latency_reverse(lp); + s->in_send_loop[indx] = 0; + } +} +/* Update the buffer space associated with this router LP */ +static void router_buf_update(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + int indx = msg->vc_index; + int output_chan = msg->output_chan; + s->vc_occupancy[indx][output_chan] -= s->params->chunk_size; + + if (s->last_buf_full[indx][output_chan] > 0.0) { + bf->c3 = 1; + msg->saved_rcv_time = s->busy_time[indx]; + msg->saved_busy_time = s->last_buf_full[indx][output_chan]; + msg->saved_sample_time = s->busy_time_sample[indx]; + s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]); + s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]); + s->last_buf_full[indx][output_chan] = 0.0; + } + if (s->queued_msgs[indx][output_chan] != NULL) { + bf->c1 = 1; + terminal_plus_message_list *head = + return_head(s->queued_msgs[indx], s->queued_msgs_tail[indx], output_chan); + router_credit_send(s, &head->msg, lp, 1); + append_to_terminal_plus_message_list(s->pending_msgs[indx], s->pending_msgs_tail[indx], output_chan, + head); + s->vc_occupancy[indx][output_chan] += s->params->chunk_size; + s->queued_count[indx] -= s->params->chunk_size; + } + if (s->in_send_loop[indx] == 0 && s->pending_msgs[indx][output_chan] != NULL) { + bf->c2 = 1; + terminal_plus_message *m; + tw_stime ts = codes_local_latency(lp); + tw_event *e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_PLUS_ROUTER, (void **) &m, NULL); + m->type = R_SEND; + m->vc_index = indx; + m->magic = router_magic_num; + s->in_send_loop[indx] = 1; + tw_event_send(e); + } + return; +} + +void router_plus_event(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + s->fwd_events++; + rc_stack_gc(lp, s->st); + + assert(msg->magic == router_magic_num); + switch (msg->type) { + case R_SEND: // Router has sent a packet to an intra-group router (local channel) + router_packet_send(s, bf, msg, lp); + break; + + case R_ARRIVE: // Router has received a packet from an intra-group router (local channel) + router_packet_receive(s, bf, msg, lp); + break; + + case R_BUFFER: + router_buf_update(s, bf, msg, lp); + break; + + default: + printf( + "\n (%lf) [Router %d] Router Message type not supported %d dest " + "terminal id %d packet ID %d ", + tw_now(lp), (int) lp->gid, msg->type, (int) msg->dest_terminal_id, (int) msg->packet_ID); + tw_error(TW_LOC, "Msg type not supported"); + break; + } +} + +/* Reverse computation handler for a terminal event */ +void terminal_plus_rc_event_handler(terminal_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + s->rev_events++; + switch (msg->type) { + case T_GENERATE: + packet_generate_rc(s, bf, msg, lp); + break; + + case T_SEND: + packet_send_rc(s, bf, msg, lp); + break; + + case T_ARRIVE: + packet_arrive_rc(s, bf, msg, lp); + break; + + case T_BUFFER: + terminal_buf_update_rc(s, bf, msg, lp); + break; + + default: + tw_error(TW_LOC, "\n Invalid terminal event type %d ", msg->type); + } +} + +/* Reverse computation handler for a router event */ +void router_plus_rc_event_handler(router_state *s, tw_bf *bf, terminal_plus_message *msg, tw_lp *lp) +{ + s->rev_events++; + + switch (msg->type) { + case R_SEND: + router_packet_send_rc(s, bf, msg, lp); + break; + case R_ARRIVE: + router_packet_receive_rc(s, bf, msg, lp); + break; + + case R_BUFFER: + router_buf_update_rc(s, bf, msg, lp); + break; + } +} + +/* dragonfly compute node and router LP types */ +extern "C" { +tw_lptype dragonfly_plus_lps[] = { + // Terminal handling functions + { + (init_f) terminal_plus_init, + (pre_run_f) NULL, + (event_f) terminal_plus_event, + (revent_f) terminal_plus_rc_event_handler, + (commit_f) NULL, + (final_f) dragonfly_plus_terminal_final, + (map_f) codes_mapping, + sizeof(terminal_state), + }, + { + (init_f) router_plus_setup, + (pre_run_f) NULL, + (event_f) router_plus_event, + (revent_f) router_plus_rc_event_handler, + (commit_f) NULL, + (final_f) dragonfly_plus_router_final, + (map_f) codes_mapping, + sizeof(router_state), + }, + {NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0}, +}; +} + +/* returns the dragonfly lp type for lp registration */ +static const tw_lptype *dragonfly_plus_get_cn_lp_type(void) +{ + return (&dragonfly_plus_lps[0]); +} +static const tw_lptype *router_plus_get_lp_type(void) +{ + return (&dragonfly_plus_lps[1]); +} + +static void dragonfly_plus_register(tw_lptype *base_type) +{ + lp_type_register(LP_CONFIG_NM_TERM, base_type); +} + +static void router_plus_register(tw_lptype *base_type) +{ + lp_type_register(LP_CONFIG_NM_ROUT, base_type); +} + +extern "C" { +/* data structure for dragonfly statistics */ +struct model_net_method dragonfly_plus_method = { + 0, + dragonfly_plus_configure, + dragonfly_plus_register, + dragonfly_plus_packet_event, + dragonfly_plus_packet_event_rc, + NULL, + NULL, + dragonfly_plus_get_cn_lp_type, + dragonfly_plus_get_msg_sz, + dragonfly_plus_report_stats, + NULL, + NULL, + NULL, //(event_f)dragonfly_plus_sample_fn, + NULL, //(revent_f)dragonfly_plus_sample_rc_fn, + (init_f) dragonfly_plus_sample_init, + NULL, //(final_f)dragonfly_plus_sample_fin +}; + +struct model_net_method dragonfly_plus_router_method = { + 0, + NULL, // handled by dragonfly_configure + router_plus_register, + NULL, + NULL, + NULL, + NULL, + router_plus_get_lp_type, + dragonfly_plus_get_msg_sz, + NULL, // not yet supported + NULL, + NULL, + NULL, //(event_f)dragonfly_plus_rsample_fn, + NULL, //(revent_f)dragonfly_plus_rsample_rc_fn, + (init_f) dragonfly_plus_rsample_init, + NULL, //(final_f)dragonfly_plus_rsample_fin +}; + +// #ifdef ENABLE_CORTEX + +// static int dragonfly_plus_get_number_of_compute_nodes(void* topo) { +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1.0; + +// // return params->total_terminals; +// } + +// static int dragonfly_plus_get_number_of_routers(void* topo) { +// // TODO +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1.0; + +// // return params->total_routers; +// } + +// static double dragonfly_plus_get_router_link_bandwidth(void* topo, router_id_t r1, router_id_t r2) { +// // TODO: handle this function for multiple cables between the routers. +// // Right now it returns the bandwidth of a single cable only. +// // Given two router ids r1 and r2, this function should return the bandwidth (double) +// // of the link between the two routers, or 0 of such a link does not exist in the topology. +// // The function should return -1 if one of the router id is invalid. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1.0; + +// // if(r1 > params->total_routers || r2 > params->total_routers) +// // return -1.0; + +// // if(r1 < 0 || r2 < 0) +// // return -1.0; + +// // int gid_r1 = r1 / params->num_routers; +// // int gid_r2 = r2 / params->num_routers; + +// // if(gid_r1 == gid_r2) +// // { +// // int lid_r1 = r1 % params->num_routers; +// // int lid_r2 = r2 % params->num_routers; + +// // /* The connection will be there if the router is in the same row or +// // * same column */ +// // int src_row_r1 = lid_r1 / params->num_router_cols; +// // int src_row_r2 = lid_r2 / params->num_router_cols; + +// // int src_col_r1 = lid_r1 % params->num_router_cols; +// // int src_col_r2 = lid_r2 % params->num_router_cols; + +// // if(src_row_r1 == src_row_r2 || src_col_r1 == src_col_r2) +// // return params->local_bandwidth; +// // else +// // return 0.0; +// // } +// // else +// // { +// // vector &curVec = interGroupLinks[r1][gid_r2]; +// // vector::iterator it = curVec.begin(); + +// // for(; it != curVec.end(); it++) +// // { +// // bLink bl = *it; +// // if(bl.dest == r2) +// // return params->global_bandwidth; +// // } + +// // return 0.0; +// // } +// // return 0.0; +// } + +// static double dragonfly_plus_get_compute_node_bandwidth(void* topo, cn_id_t node) { +// // TODO +// // Given the id of a compute node, this function should return the bandwidth of the +// // link connecting this compute node to its router. +// // The function should return -1 if the compute node id is invalid. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1.0; + +// // if(node < 0 || node >= params->total_terminals) +// // return -1.0; + +// // return params->cn_bandwidth; +// } + +// static int dragonfly_plus_get_router_neighbor_count(void* topo, router_id_t r) { +// // TODO +// // Given the id of a router, this function should return the number of routers +// // (not compute nodes) connected to it. It should return -1 if the router id +// // is not valid. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// const dragonfly_plus_param * params = &all_params[num_params-1]; +// if(!params) +// return -1.0; + +// if(r < 0 || r >= params->total_routers) +// return -1.0; + +// /* Now count the global channels */ +// set g_neighbors; + +// map< int, vector > &curMap = interGroupLinks[r]; +// map< int, vector >::iterator it = curMap.begin(); +// for(; it != curMap.end(); it++) { +// for(int l = 0; l < it->second.size(); l++) { +// g_neighbors.insert(it->second[l].dest); +// } +// } + +// int intra_group_id = r % params->num_routers; +// int num_local_neighbors; +// if (intra_group_id < (p->num_routers/2)) //leaf +// num_local_neighbors = params->num_router_spine; +// else //spine +// num_local_neighbors = params->num_router_leaf; + +// return num_local_neighbors + g_neighbors.size(); +// } + +// static void dragonfly_plus_get_router_neighbor_list(void* topo, router_id_t r, router_id_t* neighbors) { +// // Given a router id r, this function fills the "neighbors" array with the ids of routers +// // directly connected to r. It is assumed that enough memory has been allocated to "neighbors" +// // (using get_router_neighbor_count to know the required size). +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; + +// // int gid = r / params->num_routers; +// // int local_rid = r - (gid * params->num_routers); +// // int src_row = local_rid / params->num_router_cols; +// // int src_col = local_rid % params->num_router_cols; + +// // /* First the routers in the same row */ +// // int i = 0; +// // int offset = 0; +// // while(i < params->num_router_cols) +// // { +// // int neighbor = gid * params->num_routers + (src_row * params->num_router_cols) + i; +// // if(neighbor != r) +// // { +// // neighbors[offset] = neighbor; +// // offset++; +// // } +// // i++; +// // } + +// // /* Now the routers in the same column. */ +// // offset = 0; +// // i = 0; +// // while(i < params->num_router_rows) +// // { +// // int neighbor = gid * params->num_routers + src_col + (i * params->num_router_cols); + +// // if(neighbor != r) +// // { +// // neighbors[offset+params->num_router_cols-1] = neighbor; +// // offset++; +// // } +// // i++; +// // } +// // int g_offset = params->num_router_cols + params->num_router_rows - 2; + +// // /* Now fill up global channels */ +// // set g_neighbors; + +// // map< int, vector > &curMap = interGroupLinks[r]; +// // map< int, vector >::iterator it = curMap.begin(); +// // for(; it != curMap.end(); it++) { +// // for(int l = 0; l < it->second.size(); l++) { +// // g_neighbors.insert(it->second[l].dest); +// // } +// // } +// // /* Now transfer the content of the sets to the array */ +// // set::iterator it_set; +// // int count = 0; + +// // for(it_set = g_neighbors.begin(); it_set != g_neighbors.end(); it_set++) +// // { +// // neighbors[g_offset+count] = *it_set; +// // ++count; +// // } +// } + +// static int dragonfly_plus_get_router_location(void* topo, router_id_t r, int32_t* location, int size) { +// // TODO +// // Given a router id r, this function should fill the "location" array (of maximum size "size") +// // with information providing the location of the router in the topology. In a Dragonfly network, +// // for instance, this can be the array [ group_id, router_id ] where group_id is the id of the +// // group in which the router is, and router_id is the id of the router inside this group (as opposed +// // to "r" which is its global id). For a torus network, this would be the dimensions. +// // If the "size" is sufficient to hold the information, the function should return the size +// // effectively used (e.g. 2 in the above example). If however the function did not manage to use +// // the provided buffer, it should return -1. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1; + +// // if(r < 0 || r >= params->total_terminals) +// // return -1; + +// // if(size < 2) +// // return -1; + +// // int rid = r % params->num_routers; +// // int gid = r / params->num_routers; +// // location[0] = gid; +// // location[1] = rid; +// // return 2; +// } + +// static int dragonfly_plus_get_compute_node_location(void* topo, cn_id_t node, int32_t* location, int size) +// { +// // TODO +// // This function does the same as dragonfly_plus_get_router_location but for a compute node instead +// // of a router. E.g., for a dragonfly network, the location could be expressed as the array +// // [ group_id, router_id, terminal_id ] +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1; + +// // if(node < 0 || node >= params->total_terminals) +// // return -1; + +// // if(size < 3) +// // return -1; + +// // int rid = (node / params->num_cn) % params->num_routers; +// // int rid_global = node / params->num_cn; +// // int gid = rid_global / params->num_routers; +// // int lid = node % params->num_cn; + +// // location[0] = gid; +// // location[1] = rid; +// // location[2] = lid; + +// // return 3; +// } + +// static router_id_t dragonfly_plus_get_router_from_compute_node(void* topo, cn_id_t node) { +// // TODO +// // Given a node id, this function returns the id of the router connected to the node, +// // or -1 if the node id is not valid. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1; + +// // if(node < 0 || node >= params->total_terminals) +// // return -1; + +// // router_id_t rid = node / params->num_cn; +// // return rid; +// } + +// static int dragonfly_plus_get_router_compute_node_count(void* topo, router_id_t r) { +// // Given the id of a router, returns the number of compute nodes connected to this +// // router, or -1 if the router id is not valid. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; +// // if(!params) +// // return -1; + +// // if(r < 0 || r >= params->total_routers) +// // return -1; + +// // return params->num_cn; +// } + +// static void dragonfly_plus_get_router_compute_node_list(void* topo, router_id_t r, cn_id_t* nodes) { +// // TODO: What if there is an invalid router ID? +// // Given the id of a router, fills the "nodes" array with the list of ids of compute nodes +// // connected to this router. It is assumed that enough memory has been allocated for the +// // "nodes" variable to hold all the ids. +// printf("DRAGONFLY PLUS CORTEX NOT IMPLEMENTED\n"); + +// // const dragonfly_plus_param * params = &all_params[num_params-1]; + +// // for(int i = 0; i < params->num_cn; i++) +// // nodes[i] = r * params->num_cn + i; +// } + +// extern "C" { + +// cortex_topology dragonfly_plus_cortex_topology = { +// // .internal = +// NULL, +// // .get_number_of_routers = +// dragonfly_plus_get_number_of_routers, +// // .get_number_of_compute_nodes = +// dragonfly_plus_get_number_of_compute_nodes, +// // .get_router_link_bandwidth = +// dragonfly_plus_get_router_link_bandwidth, +// // .get_compute_node_bandwidth = +// dragonfly_plus_get_compute_node_bandwidth, +// // .get_router_neighbor_count = +// dragonfly_plus_get_router_neighbor_count, +// // .get_router_neighbor_list = +// dragonfly_plus_get_router_neighbor_list, +// // .get_router_location = +// dragonfly_plus_get_router_location, +// // .get_compute_node_location = +// dragonfly_plus_get_compute_node_location, +// // .get_router_from_compute_node = +// dragonfly_plus_get_router_from_compute_node, +// // .get_router_compute_node_count = +// dragonfly_plus_get_router_compute_node_count, +// // .get_router_compute_node_list = dragonfly_plus_get_router_compute_node_list, +// dragonfly_plus_get_router_compute_node_list +// }; + +// } +// #endif +} diff --git a/src/networks/model-net/model-net-lp.c b/src/networks/model-net/model-net-lp.c index 8e9677bcb5dd787d611c6ca47b3918a113c25a8b..56ae89433cba9ea469e1f47fdcf07f57bdac3485 100644 --- a/src/networks/model-net/model-net-lp.c +++ b/src/networks/model-net/model-net-lp.c @@ -140,7 +140,7 @@ void mn_event_collect(model_net_wrap_msg *m, tw_lp *lp, char *buffer, int *colle type = 9001; memcpy(buffer, &type, sizeof(type)); break; - case MN_BASE_SAMPLE: + case MN_BASE_SAMPLE: type = 9002; memcpy(buffer, &type, sizeof(type)); break; @@ -357,6 +357,10 @@ void model_net_base_configure(){ offsetof(model_net_wrap_msg, msg.m_custom_dfly); msg_offsets[DRAGONFLY_CUSTOM_ROUTER] = offsetof(model_net_wrap_msg, msg.m_custom_dfly); + msg_offsets[DRAGONFLY_PLUS] = + offsetof(model_net_wrap_msg, msg.m_dfly_plus); + msg_offsets[DRAGONFLY_PLUS_ROUTER] = + offsetof(model_net_wrap_msg, msg.m_dfly_plus); msg_offsets[SLIMFLY] = offsetof(model_net_wrap_msg, msg.m_slim); msg_offsets[FATTREE] = @@ -439,10 +443,10 @@ void model_net_base_lp_init( break; } } - + ns->nics_per_router = codes_mapping_get_lp_count(group, 1, lp_type_name, NULL, 1); - + ns->msg_id = 0; ns->next_available_time = 0; ns->node_copy_next_available_time = (tw_stime*)malloc(ns->params->node_copy_queues * sizeof(tw_stime)); @@ -596,7 +600,7 @@ void handle_new_msg( char const *sender_lpname; int rep_id, offset; model_net_request *r = &m->msg.m_base.req; - codes_mapping_get_lp_info2(r->src_lp, &sender_group, &sender_lpname, + codes_mapping_get_lp_info2(r->src_lp, &sender_group, &sender_lpname, NULL, &rep_id, &offset); num_servers = codes_mapping_get_lp_count(sender_group, 1, sender_lpname, NULL, 1); @@ -607,10 +611,10 @@ void handle_new_msg( if(!g_tw_mynode) { fprintf(stdout, "Set num_servers per router %d, servers per " "injection queue per router %d, servers per node copy queue " - "per node %d\n", num_servers, servers_per_node, + "per node %d\n", num_servers, servers_per_node, servers_per_node_queue); } - } + } if(lp->gid == m->msg.m_base.req.dest_mn_lp) { model_net_request *r = &m->msg.m_base.req; @@ -618,7 +622,7 @@ void handle_new_msg( codes_mapping_get_lp_info2(r->src_lp, NULL, NULL, NULL, &rep_id, &offset); int queue = offset/ns->nics_per_router/servers_per_node_queue; m->msg.m_base.save_ts = ns->node_copy_next_available_time[queue]; - tw_stime exp_time = ((ns->node_copy_next_available_time[queue] + tw_stime exp_time = ((ns->node_copy_next_available_time[queue] > tw_now(lp)) ? ns->node_copy_next_available_time[queue] : tw_now(lp)); exp_time += r->msg_size * codes_cn_delay; exp_time -= tw_now(lp); @@ -632,7 +636,7 @@ void handle_new_msg( tw_event *e = tw_event_new(r->final_dest_lp, exp_time, lp); memcpy(tw_event_data(e), e_msg, remote_event_size); tw_event_send(e); - e_msg = (char*)e_msg + remote_event_size; + e_msg = (char*)e_msg + remote_event_size; } if (self_event_size > 0) { exp_time += delay; @@ -658,8 +662,8 @@ void handle_new_msg( int self_event_size = r->self_event_size; if (remote_event_size > 0){ memcpy(e_new_msg, e_msg, remote_event_size); - e_msg = (char*)e_msg + remote_event_size; - e_new_msg = (char*)e_new_msg + remote_event_size; + e_msg = (char*)e_msg + remote_event_size; + e_new_msg = (char*)e_new_msg + remote_event_size; } if (self_event_size > 0){ memcpy(e_new_msg, e_msg, self_event_size); @@ -667,7 +671,7 @@ void handle_new_msg( m_new->msg.m_base.isQueueReq = 0; tw_event_send(e); return; - } + } // simply pass down to the scheduler model_net_request *r = &m->msg.m_base.req; // don't forget to set packet size, now that we're responsible for it! @@ -689,7 +693,7 @@ void handle_new_msg( if(num_servers == -1) { char const *sender_group; char const *sender_lpname; - codes_mapping_get_lp_info2(r->src_lp, &sender_group, &sender_lpname, + codes_mapping_get_lp_info2(r->src_lp, &sender_group, &sender_lpname, NULL, &rep_id, &offset); num_servers = codes_mapping_get_lp_count(sender_group, 1, sender_lpname, NULL, 1); @@ -893,7 +897,7 @@ void model_net_method_idle_event(tw_stime offset_ts, int is_recv_queue, model_net_method_idle_event2(offset_ts, is_recv_queue, 0, lp); } -void model_net_method_idle_event2(tw_stime offset_ts, int is_recv_queue, +void model_net_method_idle_event2(tw_stime offset_ts, int is_recv_queue, int queue_offset, tw_lp * lp){ tw_event *e = tw_event_new(lp->gid, offset_ts, lp); model_net_wrap_msg *m_wrap = tw_event_data(e); diff --git a/src/networks/model-net/model-net.c b/src/networks/model-net/model-net.c index 0d0beeb5dc1afdccc8832f0a3186637b4c093a6f..193752ba89d8b68827afefdda51ab4646440b380 100644 --- a/src/networks/model-net/model-net.c +++ b/src/networks/model-net/model-net.c @@ -23,6 +23,8 @@ extern struct model_net_method simplep2p_method; extern struct model_net_method torus_method; extern struct model_net_method dragonfly_method; extern struct model_net_method dragonfly_custom_method; +extern struct model_net_method dragonfly_plus_method; +extern struct model_net_method dragonfly_plus_router_method; extern struct model_net_method slimfly_method; extern struct model_net_method fattree_method; extern struct model_net_method dragonfly_router_method; @@ -45,7 +47,7 @@ char * model_net_method_names[] = { /* Global array initialization, terminated with a NULL entry */ #define X(a,b,c,d) d, -struct model_net_method* method_array[] = { +struct model_net_method* method_array[] = { NETWORK_DEF }; #undef X @@ -75,7 +77,7 @@ void model_net_register(){ for (int lpt = 0; lpt < lpgroup->lptypes_count; lpt++){ char const *nm = lpgroup->lptypes[lpt].name.ptr; for (int n = 0; n < MAX_NETS; n++){ - if (!do_config_nets[n] && + if (!do_config_nets[n] && strcmp(model_net_lp_config_names[n], nm) == 0){ do_config_nets[n] = 1; break; @@ -140,7 +142,7 @@ int* model_net_configure(int *id_count){ // init the per-msg params here memset(is_msg_params_set, 0, MAX_MN_MSG_PARAM_TYPES*sizeof(*is_msg_params_set)); - + ret = configuration_get_value_double(&config, "PARAMS", "intra_bandwidth", NULL, &cn_bandwidth); if(ret && !g_tw_mynode) { @@ -152,7 +154,7 @@ int* model_net_configure(int *id_count){ if(!g_tw_mynode) { printf("within node transfer per byte delay is %f\n", codes_cn_delay); } - + ret = configuration_get_value_int(&config, "PARAMS", "node_eager_limit", NULL, &codes_node_eager_limit); if(ret && !g_tw_mynode) { @@ -180,7 +182,7 @@ void model_net_write_stats(tw_lpid lpid, struct mn_stats* stat) char data[1024]; sprintf(id, "model-net-category-%s", stat->category); - sprintf(data, "lp:%ld\tsend_count:%ld\tsend_bytes:%ld\tsend_time:%f\t" + sprintf(data, "lp:%ld\tsend_count:%ld\tsend_bytes:%ld\tsend_time:%f\t" "recv_count:%ld\trecv_bytes:%ld\trecv_time:%f\tmax_event_size:%ld\n", (long)lpid, stat->send_count, @@ -301,9 +303,9 @@ static model_net_event_return model_net_event_impl_base( int net_id, struct codes_mctx const * send_map_ctx, struct codes_mctx const * recv_map_ctx, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, int is_pull, tw_stime offset, int remote_event_size, @@ -312,7 +314,7 @@ static model_net_event_return model_net_event_impl_base( void const * self_event, tw_lp *sender) { - if (remote_event_size + self_event_size + sizeof(model_net_wrap_msg) + if (remote_event_size + self_event_size + sizeof(model_net_wrap_msg) > g_tw_msg_sz){ tw_error(TW_LOC, "Error: model_net trying to transmit an event of size " "%d but ROSS is configured for events of size %zd\n", @@ -343,7 +345,7 @@ static model_net_event_return model_net_event_impl_base( model_net_wrap_msg *m = tw_event_data(e); msg_set_header(model_net_base_magic, MN_BASE_NEW_MSG, sender->gid, &m->h); - // set the request struct + // set the request struct model_net_request *r = &m->msg.m_base.req; r->final_dest_lp = final_dest_lp; r->dest_mn_lp = dest_mn_lp; @@ -377,14 +379,14 @@ static model_net_event_return model_net_event_impl_base( m->msg.m_base.sched_params = sched_params; else // set the default model_net_sched_set_default_params(&m->msg.m_base.sched_params); - // once params are set, clear the flags + // once params are set, clear the flags memset(is_msg_params_set, 0, MAX_MN_MSG_PARAM_TYPES*sizeof(*is_msg_params_set)); void *e_msg = (m+1); if (remote_event_size > 0){ memcpy(e_msg, remote_event, remote_event_size); - e_msg = (char*)e_msg + remote_event_size; + e_msg = (char*)e_msg + remote_event_size; } if (self_event_size > 0){ memcpy(e_msg, self_event, self_event_size); @@ -401,9 +403,9 @@ static void model_net_event_impl_base_rc(tw_lp *sender){ model_net_event_return model_net_event( int net_id, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, tw_stime offset, int remote_event_size, void const * remote_event, @@ -420,9 +422,9 @@ model_net_event_return model_net_event( model_net_event_return model_net_event_annotated( int net_id, char const * annotation, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, tw_stime offset, int remote_event_size, void const * remote_event, @@ -439,9 +441,9 @@ model_net_event_return model_net_event_mctx( int net_id, struct codes_mctx const * send_map_ctx, struct codes_mctx const * recv_map_ctx, - char const * category, - tw_lpid final_dest_lp, - uint64_t message_size, + char const * category, + tw_lpid final_dest_lp, + uint64_t message_size, tw_stime offset, int remote_event_size, void const * remote_event,