Implementation of the memory abstraction for Cooley (DDR, NVRAM and GPFS)

architectures/linux-rhel_6-x86_64/tp_memory.cpp

+#include "tp_memory.hpp"
+
+Memory::Memory ()
+{
+  this->request_ = NULL;
+  this->mmapAllocatorRank_ = 0;
+}
+
+
+Memory::~Memory ()
+{
+}
+
+
+/**********************/
+/*  |-- Allocation    */
+/**********************/
+void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) {
+  int rank, err, wrank;
+    
+  this->mem_        = mem;
+  this->masterRank_ = masterRank;
+  MPI_Comm_dup( comm, &this->comm_ );
+  this->buffSize_   = buffSize;
+    
+  MPI_Comm_rank ( this->comm_, &rank );
+  MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Allocate memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
+	this->buffer_ = malloc ( this->buffSize_ );
+	MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+      }
+      else
+	MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+	
+      MPI_Win_fence (0, this->RMAWin_ );
+      break;
+    case HDD:
+      strcpy ( this->fileName_, fileName );
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Open file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+      }
+      err = MPI_File_open( this->comm_, this->fileName_, MPI_MODE_RDWR | MPI_MODE_CREATE,
+			   MPI_INFO_NULL, &this->fileHandle_ );
+      // Preallocate the file
+      break;
+    case NVR:
+      strcpy ( this->fileName_, fileName );
+      //if ( this->mmapAllocatorRank_ == rank ) {
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Map file %s in DRAM (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	// Add test for memory capacity
+	/*
+	 *  O_RDWR | O_CREAT : read, write, create
+	 *  S_IRWXU : read, write, execute/search by owner 
+	 */
+	this->fd_ = open(this->fileName_, O_RDWR | O_CREAT, S_IRWXU);
+	
+	if ( this->fd_ == -1 ) {
+	  printMsg ( ERROR, "Error while opening the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	  MPI_Abort ( MPI_COMM_WORLD, -1 );	  
+	}
+
+	if ( ftruncate ( this->fd_, this->buffSize_ ) == -1 ) {
+	  printMsg ( ERROR, "Error while truncating the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	  MPI_Abort ( MPI_COMM_WORLD, -1 );	  
+	}
+	
+	/*
+	 *  PROT_READ | PROT_WRITE : read, write
+	 *  MAP_SHARED : Updates to the mapping are visible to other
+	 *               processes mapping the same region, and (in
+	 *               the case of file-backed mappings) are carried
+	 *               through to the underlying file. 
+	 */
+	this->buffer_ = mmap(0, this->buffSize_, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd_, 0);
+	if (this->buffer_ == MAP_FAILED) {
+	  printMsg ( ERROR, "Mmap of file %s has failed (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+          MPI_Abort ( MPI_COMM_WORLD, -1 );
+	}
+	
+	err = MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+      }
+      else
+	err = MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+
+      if ( err != MPI_SUCCESS)
+	printMsg ( ERROR, "Unable to create RMA window on mmapped file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__);
+      
+      this->memFlush ();
+      break;
+    default:
+      printMsg ( ERROR, "Unable to allocate memory (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+}
+
+
+void Memory::memFree  ( ) {
+  int rank, wrank, err;
+
+  MPI_Comm_rank ( this->comm_, &rank );
+  MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      MPI_Win_free ( &this->RMAWin_ );
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Free memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
+	free ( this->buffer_ );
+      }
+      break;
+    case HDD:
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Close file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+      }
+      MPI_File_close ( &this->fileHandle_ );
+      break;
+    case NVR:
+      //if ( this->mmapAllocatorRank_ == rank ) {
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Free memory on NVRAM and unmap file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	close ( this->fd_ );
+	err = munmap ( this->buffer_, this->buffSize_ );
+	if ( err == -1 ) {
+	  printMsg ( ERROR, "Error while unmaping the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+          MPI_Abort ( MPI_COMM_WORLD, -1 );
+	}
+	unlink ( this->fileName_ );
+      }
+      
+      err = MPI_Win_free ( &this->RMAWin_ );
+      if ( err != MPI_SUCCESS)
+        printMsg ( ERROR, "Unable to free RMA window on mmapped file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__);
+      
+      break;
+    default:
+      printMsg ( ERROR, "Unable to free memory (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+}
+
+
+/**********************/
+/*  |-- I/O           */
+/**********************/
+int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
+  int err, wrank, len;
+  char estring[MPI_MAX_ERROR_STRING];
+  MPI_Status status;
+
+  MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+    case NVR:
+      err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
+      if ( err != MPI_SUCCESS)
+        printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
+      break;
+    case HDD:
+      err = MPI_File_iwrite_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ );
+      if ( err != MPI_SUCCESS) {
+	MPI_Error_string ( err, estring, &len );
+        printMsg ( ERROR, "Error while writing data (mem = %s): %s\n", this->memName (), estring );
+      }
+      break;
+    default:
+      printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+
+  return err;
+}
+
+
+int Memory::memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
+  int err, wrank;
+  MPI_Status status;
+
+  MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+    case NVR:
+      err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
+      if ( err != MPI_SUCCESS)
+        printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
+      break;
+    case HDD:
+      err = MPI_File_iread_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ );
+      if ( err != MPI_SUCCESS)
+        printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
+      
+      break;
+    default:
+      printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+
+  return err;
+}
+
+ 
+int Memory::memFlush ( ) {
+  int err, rank, wrank;
+  MPI_Status status;
+
+  MPI_Comm_rank ( this->comm_, &rank );
+  MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
+
+  switch ( this->mem_ )
+    {
+    case DDR:
+      MPI_Win_fence ( 0, this->RMAWin_ );
+      break;
+    case HDD:
+      if ( this->request_ != NULL )
+	MPI_Wait ( &this->request_, &status );
+      break;
+    case NVR:
+      err = MPI_Win_fence ( 0, this->RMAWin_ );
+      if ( err != MPI_SUCCESS)
+	printMsg ( ERROR, "Unable to perform a synchronization on the RMA window (mem = %s)\n", this->memName () );
+      
+      //if ( this->mmapAllocatorRank_ == rank ) {
+      if ( this->masterRank_ ) {
+	printMsg ( DEBUG, "Sync memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	err = msync( this->buffer_, this->buffSize_, MS_SYNC );
+	if ( err == -1 ) {
+	  printMsg ( ERROR, "Error while syncing memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+          MPI_Abort ( MPI_COMM_WORLD, -1 );
+	}
+      }
+     
+      break;
+    default:
+      printMsg ( ERROR, "Error while flushing data (mem = %s)\n", this->memName () );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+
+  return err;
+}
+
+
+int Memory::memUncache  ( ) {
+  return 0;
+}
+
+
+/**********************/
+/*  |-- Utils         */
+/**********************/
+char* Memory::memName  ( ) {
+  switch ( this->mem_ )
+    {
+    case DDR:
+      return "DDR";
+      break;
+    case HBM:
+      return "HBM";
+      break;
+    case SSD:
+      return "SSD";
+      break;
+    case HDD:
+      return "HDD";
+      break;
+    case NVR:
+      return "NVR";
+      break;
+    default:
+      printMsg ( ERROR, "Wrong memory type!\n" );
+      MPI_Abort ( MPI_COMM_WORLD, -1 );
+    }
+}
+
+
+mem_t Memory::memType  ( char* name ) {
+  if ( ! strcmp ( "DDR", name ) ) return DDR;
+  if ( ! strcmp ( "HBM", name ) ) return HBM;
+  if ( ! strcmp ( "SSD", name ) ) return SSD;
+  if ( ! strcmp ( "HDD", name ) ) return HDD;
+  if ( ! strcmp ( "NVR", name ) ) return NVR;
+
+  printMsg ( ERROR, "Wrong memory name!\n" );
+  MPI_Abort ( MPI_COMM_WORLD, -1 );
+}
+
+
+/************************/
+/*  |-- Characteristics */
+/************************/
+int64_t Memory::memBandwidth ( ) {
+  return 0;
+}
+
+
+int64_t Memory::memLatency   ( ) {
+  return 0;
+}
+
+
+int64_t Memory::memCapacity  ( ) {
+  return 0;
+}
+
+
+bool Memory::memPersistency  ( ) {
+  return false;
+}

architectures/linux-rhel_6-x86_64/tp_memory.hpp

+#ifndef TP_MEMORY_H
+#define TP_MEMORY_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <mpi.h>
+
+#include "tp_utils.hpp"
+#include "tp_memory_interface.hpp"
+
+class Memory: public iMemory {
+public:
+  Memory ( );
+  ~Memory ( );
+  /**********************/
+  /*  |-- Allocation    */
+  /**********************/
+  void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm );
+  void memFree  ( );
+
+  /**********************/
+  /*  |-- I/O           */
+  /**********************/
+  int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank );
+  int memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ); 
+  int memFlush ( );
+  int memUncache ( );
+
+  /**********************/
+  /*  |-- Utils         */
+  /**********************/
+  char* memName  ( );
+  mem_t memType  ( char* name );
+
+  /************************/
+  /*  |-- Characteristics */
+  /************************/
+  int64_t memBandwidth   ( );
+  int64_t memLatency     ( );
+  int64_t memCapacity    ( );
+  bool    memPersistency ( );
+};
+
+#endif // TP_MEMORY_H
+