diff --git a/architectures/linux-rhel_6-ppc64/tp_network.cpp b/architectures/linux-rhel_6-ppc64/tp_network.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d6ac4b11066da731c03f8e78a91b229fc54e2472 --- /dev/null +++ b/architectures/linux-rhel_6-ppc64/tp_network.cpp @@ -0,0 +1,235 @@ +#include "tp_network.hpp" + +#define MAX_IONODES 1 + +/**********************/ +/* |-- Nodes */ +/**********************/ +int Topology::IONodeId () { + return MPIX_IO_node_id (); +} + + +int Topology::BridgeNodeId () { + return MPIX_IO_link_id (); +} + + +int Topology::ComputeNodeId () { + return 0; +} + + +int Topology::ProcessPerNode () { + MPIX_Hardware_t hw; + MPIX_Hardware(&hw); + + return hw.ppn; +} + + +int Topology::IONodesPerFile ( char* filename, int *nodesList ) { + return 1; +} + + +/**********************/ +/* |-- Network */ +/**********************/ +int Topology::NetworkDimensions () { + MPIX_Hardware_t hw; + MPIX_Hardware(&hw); + + return hw.torus_dimension; +} + + +/* |---- Coordinates */ +void Topology::RankToCoordinates ( int rank, int* coord ) { + MPIX_Rank2torus( rank, coord ); +} + + +void Topology::IONodeCoordinates ( int* coord ) { + Personality_t personality; + Kernel_GetPersonality(&personality, sizeof(personality)); + + coord[0] = personality.Network_Config.cnBridge_A; + coord[1] = personality.Network_Config.cnBridge_B; + coord[2] = personality.Network_Config.cnBridge_C; + coord[3] = personality.Network_Config.cnBridge_D; + coord[4] = personality.Network_Config.cnBridge_E; +} + + +/* |---- Distance */ +int Topology::DistanceToIONode ( int srcRank ) { + return MPIX_IO_distance (); +} + + +int Topology::DistanceBetweenRanks ( int srcRank, int destRank ) { + int srcCoords[6], destCoords[6]; + int dim, d, hops, distance; + MPIX_Hardware_t hw; + + RankToCoordinates ( srcRank, srcCoords ); + RankToCoordinates ( destRank, destCoords ); + + dim = NetworkDimensions (); + distance = 0; + MPIX_Hardware( &hw ); + + for ( d = 0; d < dim; d++ ) { + hops = abs ( destCoords[d] - srcCoords[d] ); + if ( hw.isTorus[d] == 1 ) + hops = std::min ( hops, (int)hw.Size[d] - hops ); + distance += hops; + } + + return distance; +} + + +/* |---- Routes */ +int Topology::RouteToIONode ( int srcRank, int* path ) { + int srcCoords[6], IOCoords[6]; + + RankToCoordinates ( srcRank, srcCoords ); + IONodeCoordinates ( IOCoords ); + + return RouteBetweenCoords ( srcCoords, IOCoords, path ); +} + + +int Topology::RouteBetweenRanks ( int srcRank, int destRank, int* path ) { + int srcCoords[6], destCoords[6]; + RankToCoordinates (srcRank, srcCoords); + RankToCoordinates (destRank, destCoords); + + return RouteBetweenCoords ( srcCoords, destCoords, path ); +} + + +void Topology::LinksList ( int* list ) { + return; +} + + +int Topology::getRoutingOrder ( int *ro ) { + uint64_t dcr_det_order = DCRReadUser(ND_500_DCR(CTRL_DET_ORDER)); + + int A = ND_500_DCR__CTRL_DET_ORDER__MASK0_get(dcr_det_order); + int B = ND_500_DCR__CTRL_DET_ORDER__MASK1_get(dcr_det_order); + int C = ND_500_DCR__CTRL_DET_ORDER__MASK2_get(dcr_det_order); + int D = ND_500_DCR__CTRL_DET_ORDER__MASK3_get(dcr_det_order); + int E = ND_500_DCR__CTRL_DET_ORDER__MASK4_get(dcr_det_order); + + int torus[5] = {A, B, C, D, E}; + + int index = 0; + for (int i=0; i<5 ; i++) { + if (torus[i] == 1) index = 4; + else if (torus[i] == 2) index = 3; + else if (torus[i] == 4) index = 2; + else if (torus[i] == 8) index = 1; + else if (torus[i] == 16) index = 0; + ro[i] = index; + } + + return 0; +} + + +int Topology::RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ) { + int i, interNode = 0; + int unitHop = 1; + int childCoords[6], intmdtCoords[6]; + char buf[64]; + MPIX_Hardware_t hw; + int dimSize[MPIX_TORUS_MAX_DIMS]; + int isTorus[MPIX_TORUS_MAX_DIMS]; + int *routingOrder; + int srcRank, destRank; + + MPIX_Torus2rank ( srcCoords, &srcRank ); + MPIX_Torus2rank ( destCoords, &destRank ); + + routingOrder = new int[MPIX_TORUS_MAX_DIMS]; + getRoutingOrder(routingOrder); + + MPIX_Hardware(&hw); + + for (i=0; i hw.Size[dimID]) + { + if (destCoords[dimID] > srcCoords[dimID]) + intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID]; + else + intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID]; + } + else if (abs(destCoords[dimID] - srcCoords[dimID])*2 < hw.Size[dimID]) + { + if (destCoords[dimID] < srcCoords[dimID]) + intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID]; + else + intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID]; + } + else + { + //if source coord is even, plus direction + if (srcCoords[dimID]%2 == 0) + intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID]; + //even source coord: traverse in plus direction + else + intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID]; + } + } + + ++hopnum; + + //get the rank + MPIX_Torus2rank (intmdtCoords, &intmdt_rank); + parent = intmdt_rank; + + path[interNode] = child; + interNode++; + + MPIX_Rank2torus (child, childCoords); + child = parent; + } + } + + path[interNode] = destRank; + interNode++; + + return interNode; +} diff --git a/architectures/linux-rhel_6-ppc64/tp_network.hpp b/architectures/linux-rhel_6-ppc64/tp_network.hpp index be4ad338d86b48be77d25981ce8f0c60e461baae..c4bef9d241c40ee1b7d75932d970575dc833e0af 100644 --- a/architectures/linux-rhel_6-ppc64/tp_network.hpp +++ b/architectures/linux-rhel_6-ppc64/tp_network.hpp @@ -5,246 +5,52 @@ #include #include #include -#include "tp_network_interface.hpp" - +#include #include #include #include #include #include #include +#include "tp_network_interface.hpp" #define MAX_IONODES 1 class Topology: public iTopology { public: - - int NetworkDimensions () { - MPIX_Hardware_t hw; - MPIX_Hardware(&hw); - - return hw.torus_dimension; - } - - - int ProcessPerNode () { - MPIX_Hardware_t hw; - MPIX_Hardware(&hw); - - return hw.ppn; - } - - - void RankToCoordinates ( int rank, int* coord ) { - MPIX_Rank2torus( rank, coord ); - } - - - void IONodeCoordinates ( int* coord ) { - Personality_t personality; - Kernel_GetPersonality(&personality, sizeof(personality)); - - coord[0] = personality.Network_Config.cnBridge_A; - coord[1] = personality.Network_Config.cnBridge_B; - coord[2] = personality.Network_Config.cnBridge_C; - coord[3] = personality.Network_Config.cnBridge_D; - coord[4] = personality.Network_Config.cnBridge_E; - } - - - int IONodesPerFile ( char* filename, int *nodesList ) { - return 1; - } - - - int IONodeId () { - return MPIX_IO_node_id (); - } - - - int BridgeNodeId () { - return MPIX_IO_link_id (); - } - - /***********************/ - /* Manhattan distance */ - /***********************/ - int DistanceToIONode ( int srcRank ) { - return MPIX_IO_distance (); - } - - - int DistanceBetweenRanks ( int srcRank, int destRank ) { - int srcCoords[6], destCoords[6]; - int dim, d, hops, distance; - MPIX_Hardware_t hw; - - RankToCoordinates ( srcRank, srcCoords ); - RankToCoordinates ( destRank, destCoords ); - - dim = NetworkDimensions (); - distance = 0; - MPIX_Hardware( &hw ); - - for ( d = 0; d < dim; d++ ) { - hops = abs ( destCoords[d] - srcCoords[d] ); - if ( hw.isTorus[d] == 1 ) - hops = std::min ( hops, (int)hw.Size[d] - hops ); - distance += hops; - } - - return distance; - } - - /***********************/ - /* Routes */ - /***********************/ - int RouteToIONode ( int srcRank, int* path ) { - int srcCoords[6], IOCoords[6]; - - RankToCoordinates ( srcRank, srcCoords ); - IONodeCoordinates ( IOCoords ); - - return RouteBetweenCoords ( srcCoords, IOCoords, path ); - } - - - int RouteBetweenRanks ( int srcRank, int destRank, int* path ) { - int srcCoords[6], destCoords[6]; - RankToCoordinates (srcRank, srcCoords); - RankToCoordinates (destRank, destCoords); - - return RouteBetweenCoords ( srcCoords, destCoords, path ); - } + /**********************/ + /* |-- Nodes */ + /**********************/ + int IONodeId (); + int BridgeNodeId (); + int ComputeNodeId (); + int ProcessPerNode (); + int IONodesPerFile ( char* filename, int *nodesList ); + + /**********************/ + /* |-- Network */ + /**********************/ + int NetworkDimensions (); + + /* |---- Coordinates */ + void RankToCoordinates ( int rank, int* coord ); + void IONodeCoordinates ( int* coord ); + + /* |---- Distance */ + int DistanceToIONode ( int srcRank ); + /* Minimal distance between two ranks considering a dragonfly network */ + int DistanceBetweenRanks ( int srcRank, int destRank ); + + /* |---- Routes */ + int RouteToIONode ( int srcRank, int* path ); + int RouteBetweenRanks ( int srcRank, int destRank, int* path ); + + /* |---- Links */ + void LinksList ( int* linksList ); private: - int getRoutingOrder ( int *ro ) { - uint64_t dcr_det_order = DCRReadUser(ND_500_DCR(CTRL_DET_ORDER)); - - int A = ND_500_DCR__CTRL_DET_ORDER__MASK0_get(dcr_det_order); - int B = ND_500_DCR__CTRL_DET_ORDER__MASK1_get(dcr_det_order); - int C = ND_500_DCR__CTRL_DET_ORDER__MASK2_get(dcr_det_order); - int D = ND_500_DCR__CTRL_DET_ORDER__MASK3_get(dcr_det_order); - int E = ND_500_DCR__CTRL_DET_ORDER__MASK4_get(dcr_det_order); - - int torus[5] = {A, B, C, D, E}; - - int index = 0; - for (int i=0; i<5 ; i++) { - if (torus[i] == 1) index = 4; - else if (torus[i] == 2) index = 3; - else if (torus[i] == 4) index = 2; - else if (torus[i] == 8) index = 1; - else if (torus[i] == 16) index = 0; - ro[i] = index; - } - - return 0; - } - - int RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ) { - int i, interNode = 0; - int unitHop = 1; - int childCoords[6], intmdtCoords[6]; - char buf[64]; - MPIX_Hardware_t hw; - int dimSize[MPIX_TORUS_MAX_DIMS]; - int isTorus[MPIX_TORUS_MAX_DIMS]; - int *routingOrder; - int srcRank, destRank; - - MPIX_Torus2rank ( srcCoords, &srcRank ); - MPIX_Torus2rank ( destCoords, &destRank ); - - routingOrder = new int[MPIX_TORUS_MAX_DIMS]; - getRoutingOrder(routingOrder); - - MPIX_Hardware(&hw); - - for (i=0; i hw.Size[dimID]) - { - if (destCoords[dimID] > srcCoords[dimID]) - intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID]; - else - intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID]; - } - else if (abs(destCoords[dimID] - srcCoords[dimID])*2 < hw.Size[dimID]) - { - if (destCoords[dimID] < srcCoords[dimID]) - intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID]; - else - intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID]; - } - else - { - //if source coord is even, plus direction - if (srcCoords[dimID]%2 == 0) - intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID]; - //even source coord: traverse in plus direction - else - intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID]; - } - } - - ++hopnum; - - //get the rank - MPIX_Torus2rank (intmdtCoords, &intmdt_rank); - parent = intmdt_rank; - - path[interNode] = child; - interNode++; - - MPIX_Rank2torus (child, childCoords); - child = parent; - } - } - - path[interNode] = destRank; - interNode++; - - return interNode; - } - - - void LinksList ( int* list ) { - // int nbNodes, commSize; - - // MPI_Comm_size (MPI_COMM_WORLD, &commSize); - // nbNodes = commSize / ProcessPerNode (); - - - // linksList - } + int getRoutingOrder ( int *ro ); + int RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ); }; #endif // TP_NETWORK_H