Initial commit

aggregation.hpp

+#ifndef AGGREGATION_H
+#define AGGREGATION_H
+
+#define MASTER 0
+#define LATENCY 30
+#define BANDWIDTH 1800000
+#define NBUFFERS 2
+
+#define DEBUG 1
+#define TIMING 1
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+#include <iostream>
+#include <algorithm>
+#include <vector>
+#include <map>
+#include "mpi.h"
+
+#ifdef BGQ
+#include "topology/bgq_mira.hpp"
+#endif
+
+enum MAPPING_STRATEGY
+  {
+    SHORTEST_PATH,
+    LONGEST_PATH,
+    TOPOLOGY_AWARE,
+    CONTENTION_AWARE,
+    UNIFORM
+  };
+
+enum MEMORY_LAYOUT
+  {
+    ARRAY_OF_STRUCTURES,
+    STRUCTURE_OF_ARRAYS
+  };
+
+typedef struct Round Round_t;
+struct Round
+{
+  int aggr;
+  int round;
+};
+
+  
+class Aggregation
+{
+ public:
+  Aggregation ();
+  ~Aggregation ();
+  
+  void Initialize (int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset,
+		   int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm);
+  int Commit (MPI_File fileHandle, MPI_Offset offset,
+	      void *buf, int count, MPI_Datatype datatype,
+	      MPI_Status *status, int64_t bufOffset = 0);
+  void Finalize ();
+
+  void MPIIOInfo (MPI_File fileHandle);
+
+ private:
+  /***********************/
+  /*  INITIALIZATION     */
+  /***********************/
+  void SetDefaultValues ();
+  void ParseEnvVariables ();
+  void SetCommValues ();
+  void SetOffsets ();
+  void SetNodesList ();
+
+  /***********************/
+  /*    AGGREGATION      */
+  /***********************/
+  int NumberOfAggregators ();
+  void IdentifyMyAggregators ();
+  void ElectAggregators ();
+  int64_t DataSizeSentToAggr (int aggrId);
+  void InitAggregators ();
+  void Push (MPI_File fileHandle, MPI_Status *status);
+  void iPush (MPI_File fileHandle, MPI_Request *request);
+  void GlobalFence ();
+
+  /***********************/
+  /*     PLACEMENT       */
+  /***********************/
+  int RankShortestPath (MPI_Comm aggrComm, int64_t dataSize);
+  int RankLongestPath (MPI_Comm aggrComm, int64_t dataSize);
+  int RankTopologyAware (MPI_Comm aggrComm, int64_t dataSize);
+  int RankContentionAware (MPI_Comm aggrComm, int64_t dataSize);
+  int RankUniformDistribution (MPI_Comm aggrComm, int64_t dataSize);
+  int CoordsToInt (int *coords, int dim);
+  
+  /***********************/
+  /*  OFFSET MANAGEMENT  */
+  /***********************/
+  int64_t FileToBufferOffset ();
+  int64_t BufferToFileOffset ();
+
+  /***********************/
+  /*        MISC.        */
+  /***********************/
+  const char* getStrategyName ();
+  void HandleMPIError (int retval);
+  void PrintTime (double startTime, double endTime, char* func);
+
+  /***********************/
+  /*      VARIABLES      */
+  /***********************/
+  int worldRank_;
+  int commRank_;
+  int commSize_;
+
+  int64_t rankDataSize_;
+  int64_t commDataSize_;
+
+  int64_t *chunkCount_;
+  int *chunkSize_;
+  int64_t *chunkOffset_;
+  int nChunks_;
+  int nCommit_;
+
+  int nAggr_;
+  int currentRound_;
+  int totalRounds_;
+  int roundCounter_;
+  int64_t currentDataSize_;
+
+  int intCoords_;
+  std::map<int, bool> excludedNode;
+
+  MPI_Comm subComm_;
+  int64_t offsetInFile_;
+  int64_t offsetInAggrData_;
+
+  std::vector<int> globalAggregatorsRanks;
+  std::vector<int> aggregatorsRanks;
+  std::vector<int> roundsIds;
+  std::vector<int> dataSize;
+  std::vector< std::vector<int> > chunksIndexMatching;
+
+  void *buffer1;
+  void *buffer2;
+  
+  MPI_Win RMAWin1;
+  MPI_Win RMAWin2;
+  
+  /* AGGREGATOR */
+  bool amAnAggr_;
+  int globalAggrRank_;
+  bool commSplit_;
+  MAPPING_STRATEGY strategy_;
+  MEMORY_LAYOUT layout_;
+  int64_t bufferSize_;
+  int64_t aggrDataSize_;
+  
+  int totalWrites_;
+  int writeCounter_;
+
+  bool writeDevNull_;
+  MPI_File devNullFileHandle_;
+
+  /* TIMING */
+  double startAggrTime, endAggrTime, totAggrTime;
+  double startIOTime, endIOTime, totIOTime;
+
+  Topology topology;
+};
+
+#endif /* AGGREGATION_H */

topology/bgq_mira.hpp

+#ifndef BGQ_MIRA_H
+#define BGQ_MIRA_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <algorithm>
+#include "topology.hpp"
+
+#include <spi/include/kernel/location.h>
+#include <spi/include/kernel/process.h>
+#include <spi/include/kernel/memory.h>
+#include <firmware/include/personality.h>
+#include <hwi/include/bqc/nd_500_dcr.h>
+#include <mpix.h>
+
+class Topology: public iTopology {
+public:
+
+  int NetworkDimensions () {
+    MPIX_Hardware_t hw;
+    MPIX_Hardware(&hw);
+
+    return hw.torus_dimension;
+  }
+
+
+  int ProcessPerNode () {
+    MPIX_Hardware_t hw;
+    MPIX_Hardware(&hw);
+    
+    return hw.ppn;
+  }
+
+  
+  void RankToCoordinates ( int rank, int* coord ) {
+    MPIX_Rank2torus( rank, coord );
+  }
+
+
+  void IONodeCoordinates ( int* coord ) {
+    Personality_t personality;
+    Kernel_GetPersonality(&personality, sizeof(personality));
+    
+    coord[0] = personality.Network_Config.cnBridge_A;
+    coord[1] = personality.Network_Config.cnBridge_B;
+    coord[2] = personality.Network_Config.cnBridge_C;
+    coord[3] = personality.Network_Config.cnBridge_D;
+    coord[4] = personality.Network_Config.cnBridge_E;
+  }
+
+
+  int IONodeId () {
+    return MPIX_IO_node_id ();
+  }
+
+
+  int BridgeNodeId () {
+    return MPIX_IO_link_id ();
+  }
+
+  /***********************/
+  /*  Manhattan distance */
+  /***********************/
+  int DistanceToIONode ( int srcRank ) {
+    return MPIX_IO_distance ();
+  }
+
+  
+  int DistanceBetweenRanks ( int srcRank, int destRank ) {
+    int srcCoords[6], destCoords[6];
+    int dim, d, hops, distance;
+    MPIX_Hardware_t hw;
+
+    RankToCoordinates ( srcRank, srcCoords );
+    RankToCoordinates ( destRank, destCoords );
+    
+    dim = NetworkDimensions ();
+    distance = 0;
+    MPIX_Hardware( &hw );
+    
+    for ( d = 0; d < dim; d++ ) {
+      hops = abs ( destCoords[d] - srcCoords[d] );
+      if ( hw.isTorus[d] == 1 )
+	hops = std::min ( hops, (int)hw.Size[d] - hops );
+      distance += hops;
+    }
+
+    return distance;
+  }
+
+  /***********************/
+  /*  Routes             */
+  /***********************/
+  int RouteToIONode ( int srcRank, int* path ) {
+    int srcCoords[6], IOCoords[6];
+    
+    RankToCoordinates ( srcRank, srcCoords );
+    IONodeCoordinates ( IOCoords );
+
+    return RouteBetweenCoords ( srcCoords, IOCoords, path );
+  }
+
+
+  int RouteBetweenRanks ( int srcRank, int destRank, int* path ) {
+    int srcCoords[6], destCoords[6];
+    RankToCoordinates (srcRank, srcCoords);
+    RankToCoordinates (destRank, destCoords);
+
+    return RouteBetweenCoords ( srcCoords, destCoords, path );
+  }
+
+private:
+  int getRoutingOrder ( int *ro ) {
+    uint64_t dcr_det_order = DCRReadUser(ND_500_DCR(CTRL_DET_ORDER));
+    
+    int A = ND_500_DCR__CTRL_DET_ORDER__MASK0_get(dcr_det_order);
+    int B = ND_500_DCR__CTRL_DET_ORDER__MASK1_get(dcr_det_order);
+    int C = ND_500_DCR__CTRL_DET_ORDER__MASK2_get(dcr_det_order);
+    int D = ND_500_DCR__CTRL_DET_ORDER__MASK3_get(dcr_det_order);
+    int E = ND_500_DCR__CTRL_DET_ORDER__MASK4_get(dcr_det_order);
+
+    int torus[5] = {A, B, C, D, E};
+
+    int index = 0;
+    for (int i=0; i<5 ; i++) {
+      if (torus[i] == 1)            index = 4;
+      else if (torus[i] == 2)       index = 3;
+      else if (torus[i] == 4)       index = 2;
+      else if (torus[i] == 8)       index = 1;
+      else if (torus[i] == 16)      index = 0;
+      ro[i] = index;
+    }
+
+    return 0;
+  }
+
+  int RouteBetweenCoords ( int *srcCoords, int *destCoords, int *path ) {
+    int i, interNode = 0;
+    int unitHop = 1;
+    int childCoords[6], intmdtCoords[6];
+    char buf[64];
+    MPIX_Hardware_t hw;
+    int dimSize[MPIX_TORUS_MAX_DIMS];
+    int isTorus[MPIX_TORUS_MAX_DIMS];
+    int *routingOrder;
+    int srcRank, destRank;
+
+    MPIX_Torus2rank ( srcCoords, &srcRank );
+    MPIX_Torus2rank ( destCoords, &destRank );
+
+    routingOrder = new int[MPIX_TORUS_MAX_DIMS];
+    getRoutingOrder(routingOrder);
+
+    MPIX_Hardware(&hw);
+
+    for (i=0; i<MPIX_TORUS_MAX_DIMS ; i++) {
+      isTorus[i] = hw.isTorus[i];
+      dimSize[i] = hw.Size[i];
+    }
+
+    //Initialize intermediate nodes in original path to the destination node
+    for (int dim=0; dim < MPIX_TORUS_MAX_DIMS; dim++)
+      intmdtCoords[dim] = srcCoords[dim];
+    intmdtCoords[MPIX_TORUS_MAX_DIMS] = destCoords[MPIX_TORUS_MAX_DIMS];  //T
+    
+    int hopnum = 0;
+    int hopDiff, intmdt_rank, child, parent;
+    
+    child = srcRank;
+    for (int dim=0; dim<MPIX_TORUS_MAX_DIMS; dim++) {
+
+      int dimID = routingOrder[dim];
+      hopDiff = abs(destCoords[dimID] - srcCoords[dimID]);
+
+      if (hw.isTorus[dimID] == 1)
+	hopDiff = std::min (hopDiff, (int)hw.Size[dimID] - hopDiff) ;
+
+      for(int diff=0; diff<hopDiff ;diff++) {
+	if (hw.isTorus[dimID] == 0) {
+	  if(destCoords[dimID] < srcCoords[dimID])
+	    intmdtCoords[dimID] -= unitHop;
+	  else intmdtCoords[dimID] += unitHop;
+	}
+	else {            // torus
+	  if (abs(destCoords[dimID] - srcCoords[dimID])*2 > hw.Size[dimID])
+	    {
+	      if (destCoords[dimID] > srcCoords[dimID])
+		intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
+	      else
+		intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
+	    }
+	  else if (abs(destCoords[dimID] - srcCoords[dimID])*2 < hw.Size[dimID])
+	    {
+	      if (destCoords[dimID] < srcCoords[dimID])
+		intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
+	      else
+		intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
+	    }
+	  else
+	    {
+	      //if source coord is even, plus direction
+	      if (srcCoords[dimID]%2 == 0)
+		intmdtCoords[dimID] = (intmdtCoords[dimID] + unitHop) % hw.Size[dimID];
+	      //even source coord: traverse in plus direction
+	      else
+		intmdtCoords[dimID] = ((intmdtCoords[dimID] - unitHop) + hw.Size[dimID]) % hw.Size[dimID];
+	    }
+	}
+	
+	++hopnum;
+	
+	//get the rank
+	MPIX_Torus2rank (intmdtCoords, &intmdt_rank);
+	parent = intmdt_rank;
+	
+	path[interNode] = child;
+	interNode++;
+	
+	MPIX_Rank2torus (child, childCoords);
+	child = parent;
+      }
+    }
+    
+    path[interNode] = destRank;
+    interNode++;
+    
+    return interNode;
+  }
+
+  
+  void LinksList ( int* list ) {
+  //   int nbNodes, commSize;
+
+  //   MPI_Comm_size (MPI_COMM_WORLD, &commSize);
+  //   nbNodes = commSize / ProcessPerNode ();
+
+    
+  //   linksList
+  }
+};
+
+#endif // BGQ_MIRA_H

topology/cray_xc40_theta.hpp

+#ifndef SAMPLE_H
+#define SAMPLE_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "topology.h"
+
+class Topology: public iTopology {
+public:
+  int NetworkDimensions () {
+    return 0;
+  }
+
+  int RankToCoordinates ( int rank, int* coord ) {
+    return 0;
+  }
+
+  int IONodeId () {
+    return 0;
+  }
+  int BridgeNodeId () {
+    return 0;
+  }
+
+  int DistanceToIONode () {
+    return 0;
+  }
+  
+  int DistanceToRank ( int destRank ) {
+    return 0;
+  }
+
+  int ProcessPerNode () {
+    return 0;
+  }
+};
+
+#endif // SAMPLE_H

topology/sample.hpp

+#ifndef SAMPLE_H
+#define SAMPLE_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "topology.h"
+
+class Topology: public iTopology {
+public:
+  int NetworkDimensions () {
+    return 0;
+  }
+
+  int RankToCoordinates ( int rank, int* coord ) {
+    return 0;
+  }
+
+  int IONodeId () {
+    return 0;
+  }
+  int BridgeNodeId () {
+    return 0;
+  }
+
+  int DistanceToIONode () {
+    return 0;
+  }
+  
+  int DistanceToRank ( int destRank ) {
+    return 0;
+  }
+
+  int ProcessPerNode () {
+    return 0;
+  }
+};
+
+#endif // SAMPLE_H

topology/topology.hpp

+#ifndef TOPOLOGY_H
+#define TOPOLOGY_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+class iTopology {
+ public:
+  virtual int NetworkDimensions () = 0;
+  virtual int ProcessPerNode () = 0;
+  virtual void RankToCoordinates ( int rank, int* coord ) = 0;
+  virtual void IONodeCoordinates ( int* coord ) = 0;
+  virtual int IONodeId () = 0;
+  virtual int BridgeNodeId () = 0;
+
+  virtual int DistanceToIONode ( int srcRank ) = 0;
+  virtual int DistanceBetweenRanks ( int srcRank, int destRank ) = 0;
+
+  virtual int RouteToIONode ( int srcRank, int* path ) = 0;
+  virtual int RouteBetweenRanks ( int srcRank, int destRank, int* path ) = 0;
+
+  virtual void LinksList ( int* LinksList ) = 0;
+};
+
+#endif // TOPOLOGY_H