diff --git a/architectures/linux-rhel_6-x86_64/tp_memory.cpp b/architectures/linux-rhel_6-x86_64/tp_memory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5f3fc6022a26c7404c1f687a67540d534f5294b6 --- /dev/null +++ b/architectures/linux-rhel_6-x86_64/tp_memory.cpp @@ -0,0 +1,318 @@ +#include "tp_memory.hpp" + +Memory::Memory () +{ + this->request_ = NULL; + this->mmapAllocatorRank_ = 0; +} + + +Memory::~Memory () +{ +} + + +/**********************/ +/* |-- Allocation */ +/**********************/ +void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) { + int rank, err, wrank; + + this->mem_ = mem; + this->masterRank_ = masterRank; + MPI_Comm_dup( comm, &this->comm_ ); + this->buffSize_ = buffSize; + + MPI_Comm_rank ( this->comm_, &rank ); + MPI_Comm_rank ( MPI_COMM_WORLD, &wrank ); + + switch ( this->mem_ ) + { + case DDR: + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Allocate memory on DDR (%s:%d)\n", __FILE__, __LINE__ ); + this->buffer_ = malloc ( this->buffSize_ ); + MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ ); + } + else + MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ ); + + MPI_Win_fence (0, this->RMAWin_ ); + break; + case HDD: + strcpy ( this->fileName_, fileName ); + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Open file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + } + err = MPI_File_open( this->comm_, this->fileName_, MPI_MODE_RDWR | MPI_MODE_CREATE, + MPI_INFO_NULL, &this->fileHandle_ ); + // Preallocate the file + break; + case NVR: + strcpy ( this->fileName_, fileName ); + //if ( this->mmapAllocatorRank_ == rank ) { + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Map file %s in DRAM (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + // Add test for memory capacity + /* + * O_RDWR | O_CREAT : read, write, create + * S_IRWXU : read, write, execute/search by owner + */ + this->fd_ = open(this->fileName_, O_RDWR | O_CREAT, S_IRWXU); + + if ( this->fd_ == -1 ) { + printMsg ( ERROR, "Error while opening the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + + if ( ftruncate ( this->fd_, this->buffSize_ ) == -1 ) { + printMsg ( ERROR, "Error while truncating the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + + /* + * PROT_READ | PROT_WRITE : read, write + * MAP_SHARED : Updates to the mapping are visible to other + * processes mapping the same region, and (in + * the case of file-backed mappings) are carried + * through to the underlying file. + */ + this->buffer_ = mmap(0, this->buffSize_, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd_, 0); + if (this->buffer_ == MAP_FAILED) { + printMsg ( ERROR, "Mmap of file %s has failed (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + + err = MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ ); + } + else + err = MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ ); + + if ( err != MPI_SUCCESS) + printMsg ( ERROR, "Unable to create RMA window on mmapped file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__); + + this->memFlush (); + break; + default: + printMsg ( ERROR, "Unable to allocate memory (mem = %s)\n", this->memName () ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } +} + + +void Memory::memFree ( ) { + int rank, wrank, err; + + MPI_Comm_rank ( this->comm_, &rank ); + MPI_Comm_rank ( MPI_COMM_WORLD, &wrank ); + + switch ( this->mem_ ) + { + case DDR: + MPI_Win_free ( &this->RMAWin_ ); + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Free memory on DDR (%s:%d)\n", __FILE__, __LINE__ ); + free ( this->buffer_ ); + } + break; + case HDD: + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Close file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + } + MPI_File_close ( &this->fileHandle_ ); + break; + case NVR: + //if ( this->mmapAllocatorRank_ == rank ) { + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Free memory on NVRAM and unmap file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + close ( this->fd_ ); + err = munmap ( this->buffer_, this->buffSize_ ); + if ( err == -1 ) { + printMsg ( ERROR, "Error while unmaping the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + unlink ( this->fileName_ ); + } + + err = MPI_Win_free ( &this->RMAWin_ ); + if ( err != MPI_SUCCESS) + printMsg ( ERROR, "Unable to free RMA window on mmapped file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__); + + break; + default: + printMsg ( ERROR, "Unable to free memory (mem = %s)\n", this->memName () ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } +} + + +/**********************/ +/* |-- I/O */ +/**********************/ +int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) { + int err, wrank, len; + char estring[MPI_MAX_ERROR_STRING]; + MPI_Status status; + + MPI_Comm_rank ( MPI_COMM_WORLD, &wrank ); + + switch ( this->mem_ ) + { + case DDR: + case NVR: + err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ ); + if ( err != MPI_SUCCESS) + printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () ); + break; + case HDD: + err = MPI_File_iwrite_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ ); + if ( err != MPI_SUCCESS) { + MPI_Error_string ( err, estring, &len ); + printMsg ( ERROR, "Error while writing data (mem = %s): %s\n", this->memName (), estring ); + } + break; + default: + printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + + return err; +} + + +int Memory::memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) { + int err, wrank; + MPI_Status status; + + MPI_Comm_rank ( MPI_COMM_WORLD, &wrank ); + + switch ( this->mem_ ) + { + case DDR: + case NVR: + err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ ); + if ( err != MPI_SUCCESS) + printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () ); + break; + case HDD: + err = MPI_File_iread_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ ); + if ( err != MPI_SUCCESS) + printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () ); + + break; + default: + printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + + return err; +} + + +int Memory::memFlush ( ) { + int err, rank, wrank; + MPI_Status status; + + MPI_Comm_rank ( this->comm_, &rank ); + MPI_Comm_rank ( MPI_COMM_WORLD, &wrank ); + + switch ( this->mem_ ) + { + case DDR: + MPI_Win_fence ( 0, this->RMAWin_ ); + break; + case HDD: + if ( this->request_ != NULL ) + MPI_Wait ( &this->request_, &status ); + break; + case NVR: + err = MPI_Win_fence ( 0, this->RMAWin_ ); + if ( err != MPI_SUCCESS) + printMsg ( ERROR, "Unable to perform a synchronization on the RMA window (mem = %s)\n", this->memName () ); + + //if ( this->mmapAllocatorRank_ == rank ) { + if ( this->masterRank_ ) { + printMsg ( DEBUG, "Sync memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + err = msync( this->buffer_, this->buffSize_, MS_SYNC ); + if ( err == -1 ) { + printMsg ( ERROR, "Error while syncing memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + } + + break; + default: + printMsg ( ERROR, "Error while flushing data (mem = %s)\n", this->memName () ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } + + return err; +} + + +int Memory::memUncache ( ) { + return 0; +} + + +/**********************/ +/* |-- Utils */ +/**********************/ +char* Memory::memName ( ) { + switch ( this->mem_ ) + { + case DDR: + return "DDR"; + break; + case HBM: + return "HBM"; + break; + case SSD: + return "SSD"; + break; + case HDD: + return "HDD"; + break; + case NVR: + return "NVR"; + break; + default: + printMsg ( ERROR, "Wrong memory type!\n" ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); + } +} + + +mem_t Memory::memType ( char* name ) { + if ( ! strcmp ( "DDR", name ) ) return DDR; + if ( ! strcmp ( "HBM", name ) ) return HBM; + if ( ! strcmp ( "SSD", name ) ) return SSD; + if ( ! strcmp ( "HDD", name ) ) return HDD; + if ( ! strcmp ( "NVR", name ) ) return NVR; + + printMsg ( ERROR, "Wrong memory name!\n" ); + MPI_Abort ( MPI_COMM_WORLD, -1 ); +} + + +/************************/ +/* |-- Characteristics */ +/************************/ +int64_t Memory::memBandwidth ( ) { + return 0; +} + + +int64_t Memory::memLatency ( ) { + return 0; +} + + +int64_t Memory::memCapacity ( ) { + return 0; +} + + +bool Memory::memPersistency ( ) { + return false; +} diff --git a/architectures/linux-rhel_6-x86_64/tp_memory.hpp b/architectures/linux-rhel_6-x86_64/tp_memory.hpp new file mode 100644 index 0000000000000000000000000000000000000000..bebc833621054fec3456cfcd597aa305447bb72a --- /dev/null +++ b/architectures/linux-rhel_6-x86_64/tp_memory.hpp @@ -0,0 +1,51 @@ +#ifndef TP_MEMORY_H +#define TP_MEMORY_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tp_utils.hpp" +#include "tp_memory_interface.hpp" + +class Memory: public iMemory { +public: + Memory ( ); + ~Memory ( ); + /**********************/ + /* |-- Allocation */ + /**********************/ + void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ); + void memFree ( ); + + /**********************/ + /* |-- I/O */ + /**********************/ + int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ); + int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ); + int memFlush ( ); + int memUncache ( ); + + /**********************/ + /* |-- Utils */ + /**********************/ + char* memName ( ); + mem_t memType ( char* name ); + + /************************/ + /* |-- Characteristics */ + /************************/ + int64_t memBandwidth ( ); + int64_t memLatency ( ); + int64_t memCapacity ( ); + bool memPersistency ( ); +}; + +#endif // TP_MEMORY_H +