First implementation of memory abstraction for aggregation buffers. Ongoing XP.

Makefile

@@ -12,7 +12,7 @@ libtapioca.a:$(OBJ_FILES)
 	$(MPICXX) $(MPI_CFLAGS) -c $? -o $@ $(MPI_LDFLAGS)
 
 install:
-	cp -r tapioca.hpp $(TP_INCLUDE) $(INSTALL_PATH)/include
+	cp -r tapioca.hpp tp_utils.hpp $(TP_INCLUDE) $(INSTALL_PATH)/include
 	cp libtapioca.a $(INSTALL_PATH)/lib
 
 clean:

architectures/linux-sles_12-x86_64/tp_memory.hpp

@@ -14,12 +14,12 @@ public:
   /**********************/
   /*  |-- Allocation    */
   /**********************/
-  void memAlloc ( int64_t buffSize, mem_t mem, int masterRank, char* fileName, MPI_Comm comm ) {
+  void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) {
     int rank;
     
     this->mem_        = mem;
     this->masterRank_ = masterRank;
-    this->comm_       = comm;
+    MPI_Comm_dup( comm, &this->comm_ );
     this->buffSize_   = buffSize;
     strcpy ( this->fileName_, fileName);
     
@@ -28,7 +28,7 @@ public:
     switch ( this->mem_ )
       {
       case DDR:
-	if ( rank == this->masterRank_ ) {
+	if ( this->masterRank_ ) {
 	  this->buffer_ = malloc ( this->buffSize_ );
 	  MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
 	}
@@ -51,7 +51,7 @@ public:
       {
       case DDR:
 	MPI_Win_free ( &this->RMAWin_ );
-	if ( rank == this->masterRank_ )
+	if ( this->masterRank_ )
 	  free ( this->buffer_ );
 	break;
       default:
@@ -63,14 +63,14 @@ public:
   /**********************/
   /*  |-- I/O           */
   /**********************/
-  int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset ) {
+  int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
     int err;
     MPI_Status status;
 
     switch ( this->mem_ )
       {
       case DDR:
-	err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, 0, offset, srcSize, MPI_BYTE, this->RMAWin_ );
+	err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
 	break;
       default:
 	printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
@@ -80,14 +80,14 @@ public:
     return err;
   }
 
-  int memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset ) {
+  int memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
     int err;
     MPI_Status status;
 
     switch ( this->mem_ )
       {
       case DDR:
-	err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, 0, offset, srcSize, MPI_BYTE, this->RMAWin_ );
+	err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
 	break;
       default:
 	printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );

architectures/tp_memory_interface.hpp

@@ -11,37 +11,39 @@ public:
   /************************/
   /*  |-- Allocation      */
   /************************/
-  virtual void memAlloc ( int64_t buffSize, mem_t mem, int masterRank, char* fileName, MPI_Comm comm );
-  virtual void memFree  ( );
+  virtual void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) = 0;
+  virtual void memFree  ( ) = 0;
 
   /************************/
   /*  |-- I/O             */
   /************************/
   //virtual int memWrite  ( iMemory srcMem, int64_t srcSize, int64_t offset );
   //virtual int memRead   ( iMemory srcMem, int64_t srcSize, int64_t offset );
-  virtual int memWrite  ( void* srcBuffer, int64_t srcSize, int64_t offset );
-  virtual int memRead   ( void* srcBuffer, int64_t srcSize, int64_t offset );
-  virtual int memFlush ( );
+  virtual int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) = 0;
+  virtual int memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) = 0;
+  virtual int memFlush ( ) = 0;
 
   /************************/
   /*  |-- Utils           */
   /************************/
-  virtual char* memName  ( );
-  virtual mem_t memType  ( char* name );
+  virtual char* memName  ( ) = 0;
+  virtual mem_t memType  ( char* name ) = 0;
 
   /************************/
   /*  |-- Characteristics */
   /************************/
-  virtual int64_t memBandwidth ( );
-  virtual int64_t memLatency   ( );
-  virtual int64_t memCapacity  ( );
+  virtual int64_t memBandwidth ( ) = 0;
+  virtual int64_t memLatency   ( ) = 0;
+  virtual int64_t memCapacity  ( ) = 0;
+
+  /* Temporary */
+  void *buffer_;
 
 protected:
   mem_t mem_;
-  int masterRank_;
+  bool masterRank_;
   MPI_Comm comm_;
 
-  void *buffer_;
   int64_t buffSize_;
   MPI_Win RMAWin_;
   

examples/HACC-IO/run_check_xc40.sh

 #!/bin/bash
 
-NODES=1024
+NODES=32
 PPN=16
 NPROCS=$((NODES*PPN))
 TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
 DDT="/soft/debuggers/forge/bin/ddt --connect"
-STRIPE_COUNT=48
+STRIPE_COUNT=4
 STRIPE_SIZE=8388608
 AGGR_MULT=1
 
@@ -14,7 +14,7 @@ cd $HOME/install/$ARCHI/bin/
 export TAPIOCA_DEVNULL=false
 export TAPIOCA_COMMSPLIT=true
 export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
-export TAPIOCA_NBAGGR=192
+export TAPIOCA_NBAGGR=4
 export TAPIOCA_BUFFERSIZE=16777216
 
 function setLustreFS ()
@@ -41,29 +41,23 @@ function updateSettings()
 
 function outputFile ()
 {
-    OUTPUT="$HOME/PUB-IPDPS18/dat/aggr_count"
+    OUTPUT="$HOME/PUB-IPDPS18/dat/mem_abstraction"
     OUTPUT="${OUTPUT}/HACC_XC40_${BIN}_${NODES}_${PPN}"
     OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
 }
 
-for run in {0..9}
+for run in {0..2}
 do
     for PARTICLES in 25000
     do
-	for AGGR in 48 96 144 192
-	do
-	    AGGR_MULT=$((AGGR/STRIPE_COUNT))
-	    echo "cray_cb_nodes_multiplier=$AGGR_MULT"
-	    export TAPIOCA_NBAGGR=$AGGR
-	    updateSettings
-
-	    for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
-            do
-		outputFile
-		setLustreFS >> $OUTPUT
-		aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
-		sleep 5
-	    done
+	updateSettings
+	
+	for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
+        do
+	    outputFile
+	    setLustreFS >> $OUTPUT
+	    aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
+	    sleep 5
 	done
     done
 done

tapioca.cpp

@@ -20,15 +20,9 @@ void Tapioca::Finalize ()
 
   this->commDataSize_ = 0;
 
-  MPI_Win_free (&this->RMAWin1);
-  MPI_Win_free (&this->RMAWin2);
-
+  this->memBuffer0.memFree ();
+  this->memBuffer1.memFree ();
   MPI_Comm_free (&this->subComm_);
-
-  if ( this->amAnAggr_ ) {
-    free (this->buffer1);
-    free (this->buffer2);
-  }
 }
 
 
@@ -41,10 +35,10 @@ void Tapioca::GlobalFence ()
   switch (buffer)
     {
     case 0:
-      MPI_Win_fence (0, this->RMAWin1);
+      this->memBuffer0.memFlush ();
       break;
     case 1:
-      MPI_Win_fence (0, this->RMAWin2);
+      this->memBuffer1.memFlush ();
       break;
     }
   
@@ -420,26 +414,8 @@ void Tapioca::InitAggregators ()
 {
   int aggr, retval;
   
-  if ( this->amAnAggr_ ) {
-    this->buffer1 = malloc (this->bufferSize_);
-    this->buffer2 = malloc (this->bufferSize_);
-
-    retval = MPI_Win_create (this->buffer1, this->bufferSize_, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin1);
-    this->HandleMPIError (retval);
-    MPI_Win_create (this->buffer2, this->bufferSize_, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin2);
-    this->HandleMPIError (retval);
-  }
-  else {
-    retval = MPI_Win_create (NULL, 0, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin1);
-    this->HandleMPIError (retval);
-    retval = MPI_Win_create (NULL, 0, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin2);
-    this->HandleMPIError (retval);
-  }
-
-  retval = MPI_Win_fence (0, this->RMAWin1);
-  this->HandleMPIError (retval);
-  retval = MPI_Win_fence (0, this->RMAWin2);
-  this->HandleMPIError (retval);
+  this->memBuffer0.memAlloc ( this->bufferSize_, DDR, this->amAnAggr_, NULL, this->subComm_ );
+  this->memBuffer1.memAlloc ( this->bufferSize_, DDR, this->amAnAggr_, NULL, this->subComm_ );
 
 #ifdef DEBUG
   if (this->commRank_ == MASTER) {

tapioca.hpp

@@ -19,6 +19,7 @@
 #include "mpi.h"
 
 #include "tp_network.hpp"
+#include "tp_memory.hpp"
 
 enum MAPPING_STRATEGY
   {
@@ -157,11 +158,9 @@ class Tapioca
   std::vector<int> dataSize;
   std::vector< std::vector<int> > chunksIndexMatching;
 
-  void *buffer1;
-  void *buffer2;
-  
-  MPI_Win RMAWin1;
-  MPI_Win RMAWin2;
+  Memory memBuffer0;
+  Memory memBuffer1;
+  //Memory memTarget;
   
   /* AGGREGATOR */
   bool amAnAggr_;

tp_read.cpp

@@ -130,14 +130,10 @@ int Tapioca::Read (MPI_File fileHandle, MPI_Offset offset, void *buf,
   switch (buffer)
     {
     case 0:
-      retval = MPI_Get (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
-			 targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin1);
-      this->HandleMPIError (retval);
+      this->memBuffer0.memRead ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
       break;
     case 1:
-      retval = MPI_Get (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
-			 targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin2);
-      this->HandleMPIError (retval);
+      this->memBuffer1.memRead ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
       break;
     }
  
@@ -198,11 +194,11 @@ void Tapioca::Pull (MPI_File fileHandle, MPI_Request *request)
     switch (buffer)
       {
       case 0: 
-	MPI_File_iread_at (fileHandle, offset, buffer1, dataSize, MPI_BYTE, request);
+	MPI_File_iread_at (fileHandle, offset, this->memBuffer0.buffer_, dataSize, MPI_BYTE, request);
 	MPI_Wait ( request, &status );
 	break;
       case 1:
-	MPI_File_iread_at (fileHandle, offset, buffer2, dataSize, MPI_BYTE, request);
+	MPI_File_iread_at (fileHandle, offset, this->memBuffer1.buffer_, dataSize, MPI_BYTE, request);
 	MPI_Wait ( request, &status );
 	break;
       }

tp_write.cpp

@@ -126,14 +126,10 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
   switch (buffer)
     {
     case 0:
-      retval = MPI_Put (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
-			 targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin1);
-      this->HandleMPIError (retval);
+      this->memBuffer0.memWrite ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
       break;
     case 1:
-      retval = MPI_Put (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
-			 targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin2);
-      this->HandleMPIError (retval);
+      this->memBuffer1.memWrite ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
       break;
     }
  
@@ -196,15 +192,15 @@ void Tapioca::Push (MPI_File fileHandle, MPI_Request *request)
       {
       case 0: 
 	if ( this->writeDevNull_ )
-	  MPI_File_iwrite_at (this->devNullFileHandle_, 0, buffer1, dataSize, MPI_BYTE, request);
+	  MPI_File_iwrite_at (this->devNullFileHandle_, 0, this->memBuffer0.buffer_, dataSize, MPI_BYTE, request);
 	else
-	  MPI_File_iwrite_at (fileHandle, offset, buffer1, dataSize, MPI_BYTE, request);
+	  MPI_File_iwrite_at (fileHandle, offset, this->memBuffer0.buffer_, dataSize, MPI_BYTE, request);
 	break;
       case 1:
 	if ( this->writeDevNull_ )
-	  MPI_File_iwrite_at (this->devNullFileHandle_, 0, buffer2, dataSize, MPI_BYTE, request);
+	  MPI_File_iwrite_at (this->devNullFileHandle_, 0, this->memBuffer1.buffer_, dataSize, MPI_BYTE, request);
 	else
-	  MPI_File_iwrite_at (fileHandle, offset, buffer2, dataSize, MPI_BYTE, request);
+	  MPI_File_iwrite_at (fileHandle, offset, this->memBuffer1.buffer_, dataSize, MPI_BYTE, request);
 	break;
       }