Skip to content

T
TAPIOCA
Commit a67035bf authored by Francois Tessier's avatar Francois Tessier
Browse files
Options

Move the code implementing the interconnect abstraction to cpp files and... 

Move the code implementing the interconnect abstraction to cpp files and create corresponding headers
parent 90fde14c
Hide whitespace changes
Inline Side-by-side
Showing with 268 additions and 227 deletions
architectures/linux-rhel_6-ppc64/tp_network.cpp 0 → 100644
View file @ a67035bf
#include "tp_network.hpp"
#define MAX_IONODES 1
/**********************/
/* |-- Nodes */
/**********************/
int Topology::IONodeId () {
return MPIX_IO_node_id ();
}
int Topology::BridgeNodeId () {
return MPIX_IO_link_id ();
}
int Topology::ComputeNodeId () {
return 0;
}
int Topology::ProcessPerNode () {
MPIX_Hardware_t hw;
MPIX_Hardware(&hw);
return hw.ppn;
}
int Topology::IONodesPerFile ( char* filename, int *nodesList ) {
return 1;
}
/**********************/
/* |-- Network */
/**********************/
int Topology::NetworkDimensions () {
MPIX_Hardware_t hw;
MPIX_Hardware(&hw);
return hw.torus_dimension;
}
/* |---- Coordinates */
void Topology::RankToCoordinates ( int rank, int* coord ) {
MPIX_Rank2torus( rank, coord );
}
void Topology::IONodeCoordinates ( int* coord ) {
Personality_t personality;
Kernel_GetPersonality(&personality, sizeof(personality));
coord[0] = personality.Network_Config.cnBridge_A;
coord[1] = personality.Network_Config.cnBridge_B;
coord[2] = personality.Network_Config.cnBridge_C;
coord[3] = personality.Network_Config.cnBridge_D;
coord[4] = personality.Network_Config.cnBridge_E;
}
/* |---- Distance */
int Topology::DistanceToIONode ( int srcRank ) {
return MPIX_IO_distance ();
}
int Topology::DistanceBetweenRanks ( int srcRank, int destRank ) {
int srcCoords[6], destCoords[6];
int dim, d, hops, distance;
MPIX_Hardware_t hw;
RankToCoordinates ( srcRank, srcCoords );
RankToCoordinates ( destRank, destCoords );
dim = NetworkDimensions ();
distance = 0;
MPIX_Hardware( &hw );
for ( d = 0; d < dim; d++ ) {
hops = abs ( destCoords[d] - srcCoords[d] );
if ( hw.isTorus[d] == 1 )
hops = std::min ( hops, (int)hw.Size[d] - hops );
distance += hops;
}
return distance;
}
/* |---- Routes */
int Topology::RouteToIONode ( int srcRank, int* path ) {
int srcCoords[6], IOCoords[6];
RankToCoordinates ( srcRank, srcCoords );
IONodeCoordinates ( IOCoords );
return RouteBetweenCoords ( srcCoords, IOCoords, path );
}
int Topology::RouteBetweenRanks ( int srcRank, int destRank, int* path ) {
int srcCoords[6], destCoords[6];
RankToCoordinates (srcRank, srcCoords);
RankToCoordinates (destRank, destCoords);
return RouteBetweenCoords ( srcCoords, destCoords, path );
}
void Topology::LinksList ( int* list ) {
return;
}
int Topology::getRoutingOrder ( int *ro ) {
uint64_t dcr_det_order = DCRReadUser(ND_500_DCR(CTRL_DET_ORDER));
int A = ND_500_DCR__CTRL_DET_ORDER__MASK0_get(dcr_det_order);
int B = ND_500_DCR__CTRL_DET_ORDER__MASK1_get(dcr_det_order);
int C = ND_500_DCR__CTRL_DET_ORDER__MASK2_get(dcr_det_order);
int D = ND_500_DCR__CTRL_DET_ORDER__MASK3_get(dcr_det_order);
int E = ND_500_DCR__CTRL_DET_ORDER__MASK4_get(dcr_det_order);
int torus[5] = {A, B, C, D, E};
int index = 0;
for (int i=0; i<5 ; i++) {
if (torus[i] == 1) index = 4;
else if (torus[i] == 2) index = 3;
else if (torus[i] == 4) index = 2;
else if (torus[i] == 8) index = 1;
else if (torus[i] == 16) index = 0;
ro[i] = index;
}
return 0;
}
int Topology::RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ) {
int i, interNode = 0;
int unitHop = 1;
int childCoords[6], intmdtCoords[6];
char buf[64];
MPIX_Hardware_t hw;
int dimSize[MPIX_TORUS_MAX_DIMS];
int isTorus[MPIX_TORUS_MAX_DIMS];
int *routingOrder;
int srcRank, destRank;
MPIX_Torus2rank ( srcCoords, &srcRank );
MPIX_Torus2rank ( destCoords, &destRank );
routingOrder = new int[MPIX_TORUS_MAX_DIMS];
getRoutingOrder(routingOrder);
MPIX_Hardware(&hw);
for (i=0; i<MPIX_TORUS_MAX_DIMS ; i++) {
isTorus[i] = hw.isTorus[i];
dimSize[i] = hw.Size[i];
}
//Initialize intermediate nodes in original path to the destination node
for (int dim=0; dim < MPIX_TORUS_MAX_DIMS; dim++)
intmdtCoords[dim] = srcCoords[dim];
intmdtCoords[MPIX_TORUS_MAX_DIMS] = destCoords[MPIX_TORUS_MAX_DIMS]; //T
int hopnum = 0;
int hopDiff, intmdt_rank, child, parent;
child = srcRank;
for (int dim=0; dim<MPIX_TORUS_MAX_DIMS; dim++) {
int dimID = routingOrder[dim];
hopDiff = abs(destCoords[dimID] - srcCoords[dimID]);
if (hw.isTorus[dimID] == 1)
hopDiff = std::min (hopDiff, (int)hw.Size[dimID] - hopDiff) ;
for(int diff=0; diff<hopDiff ;diff++) {
if (hw.isTorus[dimID] == 0) {
if(destCoords[dimID] < srcCoords[dimID])
intmdtCoords[dimID] -= unitHop;
else intmdtCoords[dimID] += unitHop;
}
else { // torus
if (abs(destCoords[dimID] - srcCoords[dimID])*2 > hw.Size[dimID])
{
if (destCoords[dimID] > srcCoords[dimID])
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
else
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
}
else if (abs(destCoords[dimID] - srcCoords[dimID])*2 < hw.Size[dimID])
{
if (destCoords[dimID] < srcCoords[dimID])
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
else
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
}
else
{
//if source coord is even, plus direction
if (srcCoords[dimID]%2 == 0)
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
//even source coord: traverse in plus direction
else
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
}
}
++hopnum;
//get the rank
MPIX_Torus2rank (intmdtCoords, &intmdt_rank);
parent = intmdt_rank;
path[interNode] = child;
interNode++;
MPIX_Rank2torus (child, childCoords);
child = parent;
}
}
path[interNode] = destRank;
interNode++;
return interNode;
}
architectures/linux-rhel_6-ppc64/tp_network.hpp
View file @ a67035bf
......@@ -5,246 +5,52 @@
#include <stdio.h>
#include <stdlib.h>
#include <algorithm>
#include "tp_network_interface.hpp"
#include <mpi.h>
#include <spi/include/kernel/location.h>
#include <spi/include/kernel/process.h>
#include <spi/include/kernel/memory.h>
#include <firmware/include/personality.h>
#include <hwi/include/bqc/nd_500_dcr.h>
#include <mpix.h>
#include "tp_network_interface.hpp"
#define MAX_IONODES 1
class Topology: public iTopology {
public:
int NetworkDimensions () {
MPIX_Hardware_t hw;
MPIX_Hardware(&hw);
return hw.torus_dimension;
}
int ProcessPerNode () {
MPIX_Hardware_t hw;
MPIX_Hardware(&hw);
return hw.ppn;
}
void RankToCoordinates ( int rank, int* coord ) {
MPIX_Rank2torus( rank, coord );
}
void IONodeCoordinates ( int* coord ) {
Personality_t personality;
Kernel_GetPersonality(&personality, sizeof(personality));
coord[0] = personality.Network_Config.cnBridge_A;
coord[1] = personality.Network_Config.cnBridge_B;
coord[2] = personality.Network_Config.cnBridge_C;
coord[3] = personality.Network_Config.cnBridge_D;
coord[4] = personality.Network_Config.cnBridge_E;
}
int IONodesPerFile ( char* filename, int *nodesList ) {
return 1;
}
int IONodeId () {
return MPIX_IO_node_id ();
}
int BridgeNodeId () {
return MPIX_IO_link_id ();
}
/***********************/
/* Manhattan distance */
/***********************/
int DistanceToIONode ( int srcRank ) {
return MPIX_IO_distance ();
}
int DistanceBetweenRanks ( int srcRank, int destRank ) {
int srcCoords[6], destCoords[6];
int dim, d, hops, distance;
MPIX_Hardware_t hw;
RankToCoordinates ( srcRank, srcCoords );
RankToCoordinates ( destRank, destCoords );
dim = NetworkDimensions ();
distance = 0;
MPIX_Hardware( &hw );
for ( d = 0; d < dim; d++ ) {
hops = abs ( destCoords[d] - srcCoords[d] );
if ( hw.isTorus[d] == 1 )
hops = std::min ( hops, (int)hw.Size[d] - hops );
distance += hops;
}
return distance;
}
/***********************/
/* Routes */
/***********************/
int RouteToIONode ( int srcRank, int* path ) {
int srcCoords[6], IOCoords[6];
RankToCoordinates ( srcRank, srcCoords );
IONodeCoordinates ( IOCoords );
return RouteBetweenCoords ( srcCoords, IOCoords, path );
}
int RouteBetweenRanks ( int srcRank, int destRank, int* path ) {
int srcCoords[6], destCoords[6];
RankToCoordinates (srcRank, srcCoords);
RankToCoordinates (destRank, destCoords);
return RouteBetweenCoords ( srcCoords, destCoords, path );
}
/**********************/
/* |-- Nodes */
/**********************/
int IONodeId ();
int BridgeNodeId ();
int ComputeNodeId ();
int ProcessPerNode ();
int IONodesPerFile ( char* filename, int *nodesList );
/**********************/
/* |-- Network */
/**********************/
int NetworkDimensions ();
/* |---- Coordinates */
void RankToCoordinates ( int rank, int* coord );
void IONodeCoordinates ( int* coord );
/* |---- Distance */
int DistanceToIONode ( int srcRank );
/* Minimal distance between two ranks considering a dragonfly network */
int DistanceBetweenRanks ( int srcRank, int destRank );
/* |---- Routes */
int RouteToIONode ( int srcRank, int* path );
int RouteBetweenRanks ( int srcRank, int destRank, int* path );
/* |---- Links */
void LinksList ( int* linksList );
private:
int getRoutingOrder ( int *ro ) {
uint64_t dcr_det_order = DCRReadUser(ND_500_DCR(CTRL_DET_ORDER));
int A = ND_500_DCR__CTRL_DET_ORDER__MASK0_get(dcr_det_order);
int B = ND_500_DCR__CTRL_DET_ORDER__MASK1_get(dcr_det_order);
int C = ND_500_DCR__CTRL_DET_ORDER__MASK2_get(dcr_det_order);
int D = ND_500_DCR__CTRL_DET_ORDER__MASK3_get(dcr_det_order);
int E = ND_500_DCR__CTRL_DET_ORDER__MASK4_get(dcr_det_order);
int torus[5] = {A, B, C, D, E};
int index = 0;
for (int i=0; i<5 ; i++) {
if (torus[i] == 1) index = 4;
else if (torus[i] == 2) index = 3;
else if (torus[i] == 4) index = 2;
else if (torus[i] == 8) index = 1;
else if (torus[i] == 16) index = 0;
ro[i] = index;
}
return 0;
}
int RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ) {
int i, interNode = 0;
int unitHop = 1;
int childCoords[6], intmdtCoords[6];
char buf[64];
MPIX_Hardware_t hw;
int dimSize[MPIX_TORUS_MAX_DIMS];
int isTorus[MPIX_TORUS_MAX_DIMS];
int *routingOrder;
int srcRank, destRank;
MPIX_Torus2rank ( srcCoords, &srcRank );
MPIX_Torus2rank ( destCoords, &destRank );
routingOrder = new int[MPIX_TORUS_MAX_DIMS];
getRoutingOrder(routingOrder);
MPIX_Hardware(&hw);
for (i=0; i<MPIX_TORUS_MAX_DIMS ; i++) {
isTorus[i] = hw.isTorus[i];
dimSize[i] = hw.Size[i];
}
//Initialize intermediate nodes in original path to the destination node
for (int dim=0; dim < MPIX_TORUS_MAX_DIMS; dim++)
intmdtCoords[dim] = srcCoords[dim];
intmdtCoords[MPIX_TORUS_MAX_DIMS] = destCoords[MPIX_TORUS_MAX_DIMS]; //T
int hopnum = 0;
int hopDiff, intmdt_rank, child, parent;
child = srcRank;
for (int dim=0; dim<MPIX_TORUS_MAX_DIMS; dim++) {
int dimID = routingOrder[dim];
hopDiff = abs(destCoords[dimID] - srcCoords[dimID]);
if (hw.isTorus[dimID] == 1)
hopDiff = std::min (hopDiff, (int)hw.Size[dimID] - hopDiff) ;
for(int diff=0; diff<hopDiff ;diff++) {
if (hw.isTorus[dimID] == 0) {
if(destCoords[dimID] < srcCoords[dimID])
intmdtCoords[dimID] -= unitHop;
else intmdtCoords[dimID] += unitHop;
}
else { // torus
if (abs(destCoords[dimID] - srcCoords[dimID])*2 > hw.Size[dimID])
{
if (destCoords[dimID] > srcCoords[dimID])
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
else
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
}
else if (abs(destCoords[dimID] - srcCoords[dimID])*2 < hw.Size[dimID])
{
if (destCoords[dimID] < srcCoords[dimID])
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
else
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
}
else
{
//if source coord is even, plus direction
if (srcCoords[dimID]%2 == 0)
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
//even source coord: traverse in plus direction
else
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
}
}
++hopnum;
//get the rank
MPIX_Torus2rank (intmdtCoords, &intmdt_rank);
parent = intmdt_rank;
path[interNode] = child;
interNode++;
MPIX_Rank2torus (child, childCoords);
child = parent;
}
}
path[interNode] = destRank;
interNode++;
return interNode;
}
void LinksList ( int* list ) {
// int nbNodes, commSize;
// MPI_Comm_size (MPI_COMM_WORLD, &commSize);
// nbNodes = commSize / ProcessPerNode ();
// linksList
}
int getRoutingOrder ( int *ro );
int RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path );
};
#endif // TP_NETWORK_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment