Commit d7e99d1d authored by Francois Tessier's avatar Francois Tessier

Initial commit

parent 724d227c
This diff is collapsed.
#ifndef AGGREGATION_H
#define AGGREGATION_H
#define MASTER 0
#define LATENCY 30
#define BANDWIDTH 1800000
#define NBUFFERS 2
#define DEBUG 1
#define TIMING 1
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <limits.h>
#include <float.h>
#include <iostream>
#include <algorithm>
#include <vector>
#include <map>
#include "mpi.h"
#ifdef BGQ
#include "topology/bgq_mira.hpp"
#endif
enum MAPPING_STRATEGY
{
SHORTEST_PATH,
LONGEST_PATH,
TOPOLOGY_AWARE,
CONTENTION_AWARE,
UNIFORM
};
enum MEMORY_LAYOUT
{
ARRAY_OF_STRUCTURES,
STRUCTURE_OF_ARRAYS
};
typedef struct Round Round_t;
struct Round
{
int aggr;
int round;
};
class Aggregation
{
public:
Aggregation ();
~Aggregation ();
void Initialize (int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset,
int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm);
int Commit (MPI_File fileHandle, MPI_Offset offset,
void *buf, int count, MPI_Datatype datatype,
MPI_Status *status, int64_t bufOffset = 0);
void Finalize ();
void MPIIOInfo (MPI_File fileHandle);
private:
/***********************/
/* INITIALIZATION */
/***********************/
void SetDefaultValues ();
void ParseEnvVariables ();
void SetCommValues ();
void SetOffsets ();
void SetNodesList ();
/***********************/
/* AGGREGATION */
/***********************/
int NumberOfAggregators ();
void IdentifyMyAggregators ();
void ElectAggregators ();
int64_t DataSizeSentToAggr (int aggrId);
void InitAggregators ();
void Push (MPI_File fileHandle, MPI_Status *status);
void iPush (MPI_File fileHandle, MPI_Request *request);
void GlobalFence ();
/***********************/
/* PLACEMENT */
/***********************/
int RankShortestPath (MPI_Comm aggrComm, int64_t dataSize);
int RankLongestPath (MPI_Comm aggrComm, int64_t dataSize);
int RankTopologyAware (MPI_Comm aggrComm, int64_t dataSize);
int RankContentionAware (MPI_Comm aggrComm, int64_t dataSize);
int RankUniformDistribution (MPI_Comm aggrComm, int64_t dataSize);
int CoordsToInt (int *coords, int dim);
/***********************/
/* OFFSET MANAGEMENT */
/***********************/
int64_t FileToBufferOffset ();
int64_t BufferToFileOffset ();
/***********************/
/* MISC. */
/***********************/
const char* getStrategyName ();
void HandleMPIError (int retval);
void PrintTime (double startTime, double endTime, char* func);
/***********************/
/* VARIABLES */
/***********************/
int worldRank_;
int commRank_;
int commSize_;
int64_t rankDataSize_;
int64_t commDataSize_;
int64_t *chunkCount_;
int *chunkSize_;
int64_t *chunkOffset_;
int nChunks_;
int nCommit_;
int nAggr_;
int currentRound_;
int totalRounds_;
int roundCounter_;
int64_t currentDataSize_;
int intCoords_;
std::map<int, bool> excludedNode;
MPI_Comm subComm_;
int64_t offsetInFile_;
int64_t offsetInAggrData_;
std::vector<int> globalAggregatorsRanks;
std::vector<int> aggregatorsRanks;
std::vector<int> roundsIds;
std::vector<int> dataSize;
std::vector< std::vector<int> > chunksIndexMatching;
void *buffer1;
void *buffer2;
MPI_Win RMAWin1;
MPI_Win RMAWin2;
/* AGGREGATOR */
bool amAnAggr_;
int globalAggrRank_;
bool commSplit_;
MAPPING_STRATEGY strategy_;
MEMORY_LAYOUT layout_;
int64_t bufferSize_;
int64_t aggrDataSize_;
int totalWrites_;
int writeCounter_;
bool writeDevNull_;
MPI_File devNullFileHandle_;
/* TIMING */
double startAggrTime, endAggrTime, totAggrTime;
double startIOTime, endIOTime, totIOTime;
Topology topology;
};
#endif /* AGGREGATION_H */
#ifndef BGQ_MIRA_H
#define BGQ_MIRA_H
#include <stdio.h>
#include <stdlib.h>
#include <algorithm>
#include "topology.hpp"
#include <spi/include/kernel/location.h>
#include <spi/include/kernel/process.h>
#include <spi/include/kernel/memory.h>
#include <firmware/include/personality.h>
#include <hwi/include/bqc/nd_500_dcr.h>
#include <mpix.h>
class Topology: public iTopology {
public:
int NetworkDimensions () {
MPIX_Hardware_t hw;
MPIX_Hardware(&hw);
return hw.torus_dimension;
}
int ProcessPerNode () {
MPIX_Hardware_t hw;
MPIX_Hardware(&hw);
return hw.ppn;
}
void RankToCoordinates ( int rank, int* coord ) {
MPIX_Rank2torus( rank, coord );
}
void IONodeCoordinates ( int* coord ) {
Personality_t personality;
Kernel_GetPersonality(&personality, sizeof(personality));
coord[0] = personality.Network_Config.cnBridge_A;
coord[1] = personality.Network_Config.cnBridge_B;
coord[2] = personality.Network_Config.cnBridge_C;
coord[3] = personality.Network_Config.cnBridge_D;
coord[4] = personality.Network_Config.cnBridge_E;
}
int IONodeId () {
return MPIX_IO_node_id ();
}
int BridgeNodeId () {
return MPIX_IO_link_id ();
}
/***********************/
/* Manhattan distance */
/***********************/
int DistanceToIONode ( int srcRank ) {
return MPIX_IO_distance ();
}
int DistanceBetweenRanks ( int srcRank, int destRank ) {
int srcCoords[6], destCoords[6];
int dim, d, hops, distance;
MPIX_Hardware_t hw;
RankToCoordinates ( srcRank, srcCoords );
RankToCoordinates ( destRank, destCoords );
dim = NetworkDimensions ();
distance = 0;
MPIX_Hardware( &hw );
for ( d = 0; d < dim; d++ ) {
hops = abs ( destCoords[d] - srcCoords[d] );
if ( hw.isTorus[d] == 1 )
hops = std::min ( hops, (int)hw.Size[d] - hops );
distance += hops;
}
return distance;
}
/***********************/
/* Routes */
/***********************/
int RouteToIONode ( int srcRank, int* path ) {
int srcCoords[6], IOCoords[6];
RankToCoordinates ( srcRank, srcCoords );
IONodeCoordinates ( IOCoords );
return RouteBetweenCoords ( srcCoords, IOCoords, path );
}
int RouteBetweenRanks ( int srcRank, int destRank, int* path ) {
int srcCoords[6], destCoords[6];
RankToCoordinates (srcRank, srcCoords);
RankToCoordinates (destRank, destCoords);
return RouteBetweenCoords ( srcCoords, destCoords, path );
}
private:
int getRoutingOrder ( int *ro ) {
uint64_t dcr_det_order = DCRReadUser(ND_500_DCR(CTRL_DET_ORDER));
int A = ND_500_DCR__CTRL_DET_ORDER__MASK0_get(dcr_det_order);
int B = ND_500_DCR__CTRL_DET_ORDER__MASK1_get(dcr_det_order);
int C = ND_500_DCR__CTRL_DET_ORDER__MASK2_get(dcr_det_order);
int D = ND_500_DCR__CTRL_DET_ORDER__MASK3_get(dcr_det_order);
int E = ND_500_DCR__CTRL_DET_ORDER__MASK4_get(dcr_det_order);
int torus[5] = {A, B, C, D, E};
int index = 0;
for (int i=0; i<5 ; i++) {
if (torus[i] == 1) index = 4;
else if (torus[i] == 2) index = 3;
else if (torus[i] == 4) index = 2;
else if (torus[i] == 8) index = 1;
else if (torus[i] == 16) index = 0;
ro[i] = index;
}
return 0;
}
int RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ) {
int i, interNode = 0;
int unitHop = 1;
int childCoords[6], intmdtCoords[6];
char buf[64];
MPIX_Hardware_t hw;
int dimSize[MPIX_TORUS_MAX_DIMS];
int isTorus[MPIX_TORUS_MAX_DIMS];
int *routingOrder;
int srcRank, destRank;
MPIX_Torus2rank ( srcCoords, &srcRank );
MPIX_Torus2rank ( destCoords, &destRank );
routingOrder = new int[MPIX_TORUS_MAX_DIMS];
getRoutingOrder(routingOrder);
MPIX_Hardware(&hw);
for (i=0; i<MPIX_TORUS_MAX_DIMS ; i++) {
isTorus[i] = hw.isTorus[i];
dimSize[i] = hw.Size[i];
}
//Initialize intermediate nodes in original path to the destination node
for (int dim=0; dim < MPIX_TORUS_MAX_DIMS; dim++)
intmdtCoords[dim] = srcCoords[dim];
intmdtCoords[MPIX_TORUS_MAX_DIMS] = destCoords[MPIX_TORUS_MAX_DIMS]; //T
int hopnum = 0;
int hopDiff, intmdt_rank, child, parent;
child = srcRank;
for (int dim=0; dim<MPIX_TORUS_MAX_DIMS; dim++) {
int dimID = routingOrder[dim];
hopDiff = abs(destCoords[dimID] - srcCoords[dimID]);
if (hw.isTorus[dimID] == 1)
hopDiff = std::min (hopDiff, (int)hw.Size[dimID] - hopDiff) ;
for(int diff=0; diff<hopDiff ;diff++) {
if (hw.isTorus[dimID] == 0) {
if(destCoords[dimID] < srcCoords[dimID])
intmdtCoords[dimID] -= unitHop;
else intmdtCoords[dimID] += unitHop;
}
else { // torus
if (abs(destCoords[dimID] - srcCoords[dimID])*2 > hw.Size[dimID])
{
if (destCoords[dimID] > srcCoords[dimID])
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
else
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
}
else if (abs(destCoords[dimID] - srcCoords[dimID])*2 < hw.Size[dimID])
{
if (destCoords[dimID] < srcCoords[dimID])
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
else
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
}
else
{
//if source coord is even, plus direction
if (srcCoords[dimID]%2 == 0)
intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
//even source coord: traverse in plus direction
else
intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
}
}
++hopnum;
//get the rank
MPIX_Torus2rank (intmdtCoords, &intmdt_rank);
parent = intmdt_rank;
path[interNode] = child;
interNode++;
MPIX_Rank2torus (child, childCoords);
child = parent;
}
}
path[interNode] = destRank;
interNode++;
return interNode;
}
void LinksList ( int* list ) {
// int nbNodes, commSize;
// MPI_Comm_size (MPI_COMM_WORLD, &commSize);
// nbNodes = commSize / ProcessPerNode ();
// linksList
}
};
#endif // BGQ_MIRA_H
#ifndef SAMPLE_H
#define SAMPLE_H
#include <stdio.h>
#include <stdlib.h>
#include "topology.h"
class Topology: public iTopology {
public:
int NetworkDimensions () {
return 0;
}
int RankToCoordinates ( int rank, int* coord ) {
return 0;
}
int IONodeId () {
return 0;
}
int BridgeNodeId () {
return 0;
}
int DistanceToIONode () {
return 0;
}
int DistanceToRank ( int destRank ) {
return 0;
}
int ProcessPerNode () {
return 0;
}
};
#endif // SAMPLE_H
#ifndef SAMPLE_H
#define SAMPLE_H
#include <stdio.h>
#include <stdlib.h>
#include "topology.h"
class Topology: public iTopology {
public:
int NetworkDimensions () {
return 0;
}
int RankToCoordinates ( int rank, int* coord ) {
return 0;
}
int IONodeId () {
return 0;
}
int BridgeNodeId () {
return 0;
}
int DistanceToIONode () {
return 0;
}
int DistanceToRank ( int destRank ) {
return 0;
}
int ProcessPerNode () {
return 0;
}
};
#endif // SAMPLE_H
#ifndef TOPOLOGY_H
#define TOPOLOGY_H
#include <stdio.h>
#include <stdlib.h>
class iTopology {
public:
virtual int NetworkDimensions () = 0;
virtual int ProcessPerNode () = 0;
virtual void RankToCoordinates ( int rank, int* coord ) = 0;
virtual void IONodeCoordinates ( int* coord ) = 0;
virtual int IONodeId () = 0;
virtual int BridgeNodeId () = 0;
virtual int DistanceToIONode ( int srcRank ) = 0;
virtual int DistanceBetweenRanks ( int srcRank, int destRank ) = 0;
virtual int RouteToIONode ( int srcRank, int* path ) = 0;
virtual int RouteBetweenRanks ( int srcRank, int destRank, int* path ) = 0;
virtual void LinksList ( int* LinksList ) = 0;
};
#endif // TOPOLOGY_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment