Commit 17d938e3 authored by Francois Tessier's avatar Francois Tessier

Implementation of the memory abstraction for Cooley (DDR, NVRAM and GPFS)

parent 8b8b2478
#include "tp_memory.hpp"
Memory::Memory ()
{
this->request_ = NULL;
this->mmapAllocatorRank_ = 0;
}
Memory::~Memory ()
{
}
/**********************/
/* |-- Allocation */
/**********************/
void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) {
int rank, err, wrank;
this->mem_ = mem;
this->masterRank_ = masterRank;
MPI_Comm_dup( comm, &this->comm_ );
this->buffSize_ = buffSize;
MPI_Comm_rank ( this->comm_, &rank );
MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
switch ( this->mem_ )
{
case DDR:
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Allocate memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
this->buffer_ = malloc ( this->buffSize_ );
MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
}
else
MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
MPI_Win_fence (0, this->RMAWin_ );
break;
case HDD:
strcpy ( this->fileName_, fileName );
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Open file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
}
err = MPI_File_open( this->comm_, this->fileName_, MPI_MODE_RDWR | MPI_MODE_CREATE,
MPI_INFO_NULL, &this->fileHandle_ );
// Preallocate the file
break;
case NVR:
strcpy ( this->fileName_, fileName );
//if ( this->mmapAllocatorRank_ == rank ) {
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Map file %s in DRAM (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
// Add test for memory capacity
/*
* O_RDWR | O_CREAT : read, write, create
* S_IRWXU : read, write, execute/search by owner
*/
this->fd_ = open(this->fileName_, O_RDWR | O_CREAT, S_IRWXU);
if ( this->fd_ == -1 ) {
printMsg ( ERROR, "Error while opening the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
if ( ftruncate ( this->fd_, this->buffSize_ ) == -1 ) {
printMsg ( ERROR, "Error while truncating the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
/*
* PROT_READ | PROT_WRITE : read, write
* MAP_SHARED : Updates to the mapping are visible to other
* processes mapping the same region, and (in
* the case of file-backed mappings) are carried
* through to the underlying file.
*/
this->buffer_ = mmap(0, this->buffSize_, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd_, 0);
if (this->buffer_ == MAP_FAILED) {
printMsg ( ERROR, "Mmap of file %s has failed (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
err = MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
}
else
err = MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
if ( err != MPI_SUCCESS)
printMsg ( ERROR, "Unable to create RMA window on mmapped file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__);
this->memFlush ();
break;
default:
printMsg ( ERROR, "Unable to allocate memory (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
void Memory::memFree ( ) {
int rank, wrank, err;
MPI_Comm_rank ( this->comm_, &rank );
MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
switch ( this->mem_ )
{
case DDR:
MPI_Win_free ( &this->RMAWin_ );
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Free memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
free ( this->buffer_ );
}
break;
case HDD:
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Close file %s on HDD (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
}
MPI_File_close ( &this->fileHandle_ );
break;
case NVR:
//if ( this->mmapAllocatorRank_ == rank ) {
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Free memory on NVRAM and unmap file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
close ( this->fd_ );
err = munmap ( this->buffer_, this->buffSize_ );
if ( err == -1 ) {
printMsg ( ERROR, "Error while unmaping the file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
unlink ( this->fileName_ );
}
err = MPI_Win_free ( &this->RMAWin_ );
if ( err != MPI_SUCCESS)
printMsg ( ERROR, "Unable to free RMA window on mmapped file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__);
break;
default:
printMsg ( ERROR, "Unable to free memory (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
/**********************/
/* |-- I/O */
/**********************/
int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
int err, wrank, len;
char estring[MPI_MAX_ERROR_STRING];
MPI_Status status;
MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
switch ( this->mem_ )
{
case DDR:
case NVR:
err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
if ( err != MPI_SUCCESS)
printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
break;
case HDD:
err = MPI_File_iwrite_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ );
if ( err != MPI_SUCCESS) {
MPI_Error_string ( err, estring, &len );
printMsg ( ERROR, "Error while writing data (mem = %s): %s\n", this->memName (), estring );
}
break;
default:
printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
return err;
}
int Memory::memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
int err, wrank;
MPI_Status status;
MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
switch ( this->mem_ )
{
case DDR:
case NVR:
err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
if ( err != MPI_SUCCESS)
printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
break;
case HDD:
err = MPI_File_iread_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &this->request_ );
if ( err != MPI_SUCCESS)
printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
break;
default:
printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
return err;
}
int Memory::memFlush ( ) {
int err, rank, wrank;
MPI_Status status;
MPI_Comm_rank ( this->comm_, &rank );
MPI_Comm_rank ( MPI_COMM_WORLD, &wrank );
switch ( this->mem_ )
{
case DDR:
MPI_Win_fence ( 0, this->RMAWin_ );
break;
case HDD:
if ( this->request_ != NULL )
MPI_Wait ( &this->request_, &status );
break;
case NVR:
err = MPI_Win_fence ( 0, this->RMAWin_ );
if ( err != MPI_SUCCESS)
printMsg ( ERROR, "Unable to perform a synchronization on the RMA window (mem = %s)\n", this->memName () );
//if ( this->mmapAllocatorRank_ == rank ) {
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Sync memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
err = msync( this->buffer_, this->buffSize_, MS_SYNC );
if ( err == -1 ) {
printMsg ( ERROR, "Error while syncing memory and file %s (%s:%d)\n", this->fileName_, __FILE__, __LINE__ );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
break;
default:
printMsg ( ERROR, "Error while flushing data (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
return err;
}
int Memory::memUncache ( ) {
return 0;
}
/**********************/
/* |-- Utils */
/**********************/
char* Memory::memName ( ) {
switch ( this->mem_ )
{
case DDR:
return "DDR";
break;
case HBM:
return "HBM";
break;
case SSD:
return "SSD";
break;
case HDD:
return "HDD";
break;
case NVR:
return "NVR";
break;
default:
printMsg ( ERROR, "Wrong memory type!\n" );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
mem_t Memory::memType ( char* name ) {
if ( ! strcmp ( "DDR", name ) ) return DDR;
if ( ! strcmp ( "HBM", name ) ) return HBM;
if ( ! strcmp ( "SSD", name ) ) return SSD;
if ( ! strcmp ( "HDD", name ) ) return HDD;
if ( ! strcmp ( "NVR", name ) ) return NVR;
printMsg ( ERROR, "Wrong memory name!\n" );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
/************************/
/* |-- Characteristics */
/************************/
int64_t Memory::memBandwidth ( ) {
return 0;
}
int64_t Memory::memLatency ( ) {
return 0;
}
int64_t Memory::memCapacity ( ) {
return 0;
}
bool Memory::memPersistency ( ) {
return false;
}
#ifndef TP_MEMORY_H
#define TP_MEMORY_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <mpi.h>
#include "tp_utils.hpp"
#include "tp_memory_interface.hpp"
class Memory: public iMemory {
public:
Memory ( );
~Memory ( );
/**********************/
/* |-- Allocation */
/**********************/
void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm );
void memFree ( );
/**********************/
/* |-- I/O */
/**********************/
int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank );
int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank );
int memFlush ( );
int memUncache ( );
/**********************/
/* |-- Utils */
/**********************/
char* memName ( );
mem_t memType ( char* name );
/************************/
/* |-- Characteristics */
/************************/
int64_t memBandwidth ( );
int64_t memLatency ( );
int64_t memCapacity ( );
bool memPersistency ( );
};
#endif // TP_MEMORY_H
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment