Implement the memory abstraction for BG/Q (DDR and HDD)

architectures/linux-rhel_6-ppc64/tp_memory.cpp

+#include "tp_memory.hpp"
+
+Memory::Memory ()
+{
+  this->request_ = NULL;
+  this->mmapAllocatorRank_ = 0;
+}
+
+
+Memory::~Memory ()
+{
+}
+
+
+/**********************/
+/*  |-- Allocation    */
+/**********************/
+void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) {
+  int rank, err;
+    
+  this->mem_        = mem;
+  this->masterRank_ = masterRank;
+  MPI_Comm_dup( comm, &this->comm_ );
+  this->buffSize_   = buffSize;
+    
+  MPI_Comm_rank ( this->comm_, &rank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Allocate memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
+	this->buffer_ = malloc ( this->buffSize_ );
+	MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+      }
+      else
+	MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+	
+      MPI_Win_fence (0, this->RMAWin_ );
+      break;
+    case HDD:
+      strcpy ( this->fileName_, fileName );
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Open file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+      }
+      err = MPI_File_open( this->comm_, this->fileName_, MPI_MODE_RDWR | MPI_MODE_CREATE,
+			   MPI_INFO_NULL, &this->fileHandle_ );
+      // Preallocate the file
+      break;
+    default:
+      printMsg ( ERROR, "Unable to allocate memory (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+}
+
+
+void Memory::memFree  ( ) {
+  int rank, err;
+  MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
+    
+  switch ( this->mem_ )
+    {
+    case DDR:
+      MPI_Win_free ( &this->RMAWin_ );
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Free memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
+	free ( this->buffer_ );
+      }
+      break;
+    case HDD:
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Close file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+      }
+      MPI_File_close ( &this->fileHandle_ );
+      break;
+    default:
+      printMsg ( ERROR, "Unable to free memory (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+}
+
+
+/**********************/
+/*  |-- I/O           */
+/**********************/
+int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
+  int err;
+  MPI_Status status;
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
+      break;
+    case HDD:
+      err = MPI_File_iwrite_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ );
+      break;
+    default:
+      printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+
+  return err;
+}
+
+
+int Memory::memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
+  int err;
+  MPI_Status status;
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
+      break;
+    case HDD:
+      err = MPI_File_iread_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ );
+      break;
+    default:
+      printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+
+  return err;
+}
+
+ 
+int Memory::memFlush ( ) {
+  int err, rank;
+  MPI_Status status;
+
+  MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      MPI_Win_fence ( 0, this->RMAWin_ );
+      break;
+    case HDD:
+      if ( this->request_ != NULL )
+	MPI_Wait ( &this->request_, &status );
+      break;
+    default:
+      printMsg ( ERROR, "Error while flushing data (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+
+  return err;
+}
+
+
+int Memory::memUncache  ( ) {
+  return 0;
+}
+
+
+/**********************/
+/*  |-- Utils         */
+/**********************/
+char* Memory::memName  ( ) {
+  switch ( this->mem_ )
+    {
+    case DDR:
+      return "DDR";
+      break;
+    case HDD:
+      return "HDD";
+      break;
+    default:
+      printMsg ( ERROR, "Wrong memory type!\n" );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+}
+
+
+mem_t Memory::memType  ( char* name ) {
+  if ( ! strcmp ( "DDR", name ) ) return DDR;
+  if ( ! strcmp ( "HDD", name ) ) return HDD;
+
+  printMsg ( ERROR, "Wrong memory name!\n" );
+  MPI_Abort ( MPI_COMM_WORLD, -1 );
+}
+
+
+/************************/
+/*  |-- Characteristics */
+/************************/
+int64_t Memory::memBandwidth ( ) {
+  return 0;
+}
+
+
+int64_t Memory::memLatency   ( ) {
+  return 0;
+}
+
+
+int64_t Memory::memCapacity  ( ) {
+  return 0;
+}
+
+
+bool Memory::memPersistency  ( ) {
+  return false;
+}

architectures/linux-rhel_6-ppc64/tp_memory.hpp

@@ -3,63 +3,48 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <mpix.h>
-
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <mpi.h>
+
+#include "tp_utils.hpp"
 #include "tp_memory_interface.hpp"
 
 class Memory: public iMemory {
 public:
+  Memory ( );
+  ~Memory ( );
   /**********************/
   /*  |-- Allocation    */
   /**********************/
-  void memAlloc ( union membuffer *buff, int64_t buffSize, mem_t mem, char* fileName ) {
-    return;
-  }
-
-  void memFree  ( union membuffer *buff, mem_t mem ) {
-    return;
-  }
+  void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm );
+  void memFree  ( );
 
   /**********************/
   /*  |-- I/O           */
   /**********************/
-  int memWrite ( void* srcBuffer, int64_t srcSize, union membuffer *destBuffer, mem_t mem, int64_t offset ) {
-    return 0;
-  }
-
-  int memRead  ( void* srcBuffer, int64_t srcSize, union membuffer *destBuffer, mem_t mem, int64_t offset ) {
-    return 0;
-  }
- 
-  void  memFlush ( union membuffer *buff, mem_t mem ) {
-    return;
-  }
+  int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank );
+  int memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ); 
+  int memFlush ( );
+  int memUncache ( );
 
   /**********************/
   /*  |-- Utils         */
   /**********************/
-  char* memName  ( mem_t mem ) {
-    return NULL;
-  }
-
-  mem_t memType  ( char* name ) {
-    return NULL;
-  }
+  char* memName  ( );
+  mem_t memType  ( char* name );
 
   /************************/
   /*  |-- Characteristics */
   /************************/
-  int64_t memBandwidth ( mem_t mem ) {
-    return 0;
-  }
-
-  int64_t memLatency   ( mem_t mem ) {
-    return 0;
-  }
-
-  int64_t memCapacity  ( mem_t mem ) {
-    return 0;
-  }
+  int64_t memBandwidth   ( );
+  int64_t memLatency     ( );
+  int64_t memCapacity    ( );
+  bool    memPersistency ( );
 };
 
 #endif // TP_MEMORY_H