Draft of the NVRAM implementation of the abstraction layer for XC40

architectures/linux-sles_12-x86_64/tp_memory.cpp

@@ -3,6 +3,7 @@
 Memory::Memory ()
 {
   this->request_ = NULL;
+  this->mmapAllocatorRank_ = 0;
 }
 
 
@@ -55,6 +56,37 @@ void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* file
       }
       err = MPI_File_open( this->comm_, this->fileName_, MPI_MODE_RDWR | MPI_MODE_CREATE,
 			   MPI_INFO_NULL, &this->fileHandle_ );
+      // Preallocate the file
+      break;
+    case NVR:
+      strcpy ( this->fileName_, fileName );
+      if ( this->mmapAllocatorRank_ == rank ) {
+	printMsg ( DEBUG, "Map file %s in DRAM (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	// Add test for memory capacity
+	this->fd_ = open(this->fileName_, O_RDWR | O_CREAT, S_IRWXU);
+	
+	if ( this->fd_ == -1 ) {
+	  printMsg ( ERROR, "Error while opening the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	  MPI_Abort ( MPI_COMM_WORLD, -1 );	  
+	}
+
+	if ( ftruncate ( this->fd_, this->buffSize_ ) == -1 ) {
+	  printMsg ( ERROR, "Error while truncating the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	  MPI_Abort ( MPI_COMM_WORLD, -1 );	  
+	}
+	
+	this->ptrMap_ = mmap(0, this->buffSize_, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd_, 0);
+	if (this->ptrMap_ == MAP_FAILED) {
+	  printMsg ( ERROR, "Mmap of file %s has failed (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+          MPI_Abort ( MPI_COMM_WORLD, -1 );
+	}
+	
+	MPI_Win_create ( this->ptrMap_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+      }
+      else
+	MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
+
+      MPI_Win_fence (0, this->RMAWin_ );
       break;
     default:
       printMsg ( ERROR, "Unable to allocate memory (mem = %s)\n", this->memName () );
@@ -64,7 +96,7 @@ void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* file
 
 
 void Memory::memFree  ( ) {
-  int rank;
+  int rank, err;
   MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
     
   switch ( this->mem_ )
@@ -89,6 +121,20 @@ void Memory::memFree  ( ) {
       }
       MPI_File_close ( &this->fileHandle_ );
       break;
+    case NVR:
+      this->memFlush ();
+      MPI_Win_free ( &this->RMAWin_ );
+      
+      if ( this->mmapAllocatorRank_ == rank ) {
+	printMsg ( DEBUG, "Free memory on NVRAM and unmap file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	err = munmap ( this->ptrMap_, this->buffSize_ );
+	if ( err == -1 ) {
+	  printMsg ( ERROR, "Error while unmaping the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+          MPI_Abort ( MPI_COMM_WORLD, -1 );
+	}
+	close ( this->fd_ );
+      }
+      break;
     default:
       printMsg ( ERROR, "Unable to free memory (mem = %s)\n", this->memName () );
       MPI_Abort ( MPI_COMM_WORLD, -1 );
@@ -107,6 +153,7 @@ int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
     {
     case DDR:
     case HBM:
+    case NVR:
       err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
       break;
     case HDD:
@@ -129,6 +176,7 @@ int Memory::memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
     {
     case DDR:
     case HBM:
+    case NVR:
       err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
       break;
     case HDD:
@@ -144,9 +192,11 @@ int Memory::memRead  ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
 
  
 int Memory::memFlush ( ) {
-  int err;
+  int err, rank;
   MPI_Status status;
 
+  MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
+
   switch ( this->mem_ )
     {
     case DDR:
@@ -157,6 +207,17 @@ int Memory::memFlush ( ) {
       if ( this->request_ != NULL )
 	MPI_Wait ( &this->request_, &status );
       break;
+    case NVR:
+      if ( this->mmapAllocatorRank_ == rank ) {
+	printMsg ( DEBUG, "Sync memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+	err = msync( this->ptrMap_, this->buffSize_, MS_SYNC );
+	if ( err == -1 ) {
+	  printMsg ( ERROR, "Error while syncing memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
+          MPI_Abort ( MPI_COMM_WORLD, -1 );
+	}
+      }
+      //MPI_Win_fence ( 0, this->RMAWin_ );
+      break;
     default:
       printMsg ( ERROR, "Error while flushing data (mem = %s)\n", this->memName () );
       MPI_Abort ( MPI_COMM_WORLD, -1 );
@@ -189,6 +250,9 @@ char* Memory::memName  ( ) {
     case HDD:
       return "HDD";
       break;
+    case NVR:
+      return "NVR";
+      break;
     default:
       printMsg ( ERROR, "Wrong memory type!\n" );
       MPI_Abort ( MPI_COMM_WORLD, -1 );
@@ -201,6 +265,7 @@ mem_t Memory::memType  ( char* name ) {
   if ( ! strcmp ( "HBM", name ) ) return HBM;
   if ( ! strcmp ( "SSD", name ) ) return SSD;
   if ( ! strcmp ( "HDD", name ) ) return HDD;
+  if ( ! strcmp ( "NVR", name ) ) return NVR;
 
   printMsg ( ERROR, "Wrong memory name!\n" );
   MPI_Abort ( MPI_COMM_WORLD, -1 );

architectures/linux-sles_12-x86_64/tp_memory.hpp

@@ -4,6 +4,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <mpi.h>
 #include <hbwmalloc.h>