Commit d48d7184 authored by Francois Tessier's avatar Francois Tessier

Start implementing the RAN aggregation layer

parent e78dce67
......@@ -35,9 +35,15 @@ void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* file
}
else
MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
MPI_Win_fence (0, this->RMAWin_ );
break;
case NAM:
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Connect to the RAN volume (%s:%d)\n", __FILE__, __LINE__ );
}
kdsa_connect(this->RAN_volume_, 0, &this->RAN_handle_);
break;
case NLS:
case PFS:
char *name;
......@@ -121,6 +127,9 @@ void Memory::memFree ( ) {
free ( this->buffer_ );
}
break;
case NAM:
kdsa_disconnect (this->RAN_handle_);
break;
case NLS:
case PFS:
if ( this->masterRank_ ) {
......@@ -164,6 +173,12 @@ int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
case NVR:
err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
break;
case NAM:
kdsa_mem_key_t key;
kdsa_register_mem (this->RAN_handle_, srcBuffer, srcSize, &key);
kdsa_write (this->RAN_handle_, key, offset, srcBuffer, srcSize);
kdsa_deregister_mem(key);
break;
case NLS:
case PFS:
err = MPI_File_write_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &status );
......@@ -187,6 +202,12 @@ int Memory::memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
case NVR:
err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
break;
case NAM:
kdsa_mem_key_t key;
kdsa_register_mem (this->RAN_handle_, srcBuffer, srcSize, &key);
kdsa_read (this->RAN_handle_, key, offset, srcBuffer, srcSize);
kdsa_deregister_mem(key);
break;
case NLS:
case PFS:
err = MPI_File_read_at ( this->fileHandle_, offset, srcBuffer, srcSize, MPI_BYTE, &status );
......@@ -211,6 +232,9 @@ int Memory::memFlush ( ) {
case DDR:
MPI_Win_fence ( 0, this->RMAWin_ );
break;
case NAM:
kdsa_memory_barrier (this->RAN_handle_);
break;
case NLS:
case PFS:
if ( this->request_ != MPI_REQUEST_NULL )
......@@ -260,6 +284,9 @@ char* Memory::memName ( mem_t mem ) {
case DDR:
return "DDR";
break;
case NAM:
return "NAM";
break;
case PFS:
return "PFS";
break;
......@@ -278,6 +305,7 @@ char* Memory::memName ( mem_t mem ) {
mem_t Memory::memTypeByName ( char* name ) {
if ( ! strcmp ( "DDR", name ) ) return DDR;
if ( ! strcmp ( "NAM", name ) ) return NAM;
if ( ! strcmp ( "PFS", name ) ) return PFS;
if ( ! strcmp ( "NLS", name ) ) return NLS;
if ( ! strcmp ( "NVR", name ) ) return NVR;
......@@ -336,6 +364,10 @@ int64_t Memory::memBandwidth ( mem_t mem ) {
case DDR:
return 90000000;
break;
case NAM:
/* TODO: Memory bw? Limited by the IB network? */
return 1800000;
break;
case PFS:
return 1800000;
break;
......@@ -360,6 +392,10 @@ int64_t Memory::memLatency ( mem_t mem ) {
case DDR:
return 2;
break;
case NAM:
/* TODO: Latency? */
return 30;
break;
case PFS:
return 30;
break;
......@@ -382,6 +418,9 @@ int64_t Memory::memCapacity ( mem_t mem ) {
case DDR:
return 384000000000;
break;
case NAM:
return LONG_MAX;
break;
case PFS:
return LONG_MAX;
break;
......
......@@ -12,6 +12,7 @@
#include <fcntl.h>
#include <mpi.h>
#include "kdsa.h"
#include "tp_utils.hpp"
#include "tp_memory_interface.hpp"
......@@ -56,6 +57,8 @@ public:
bool memPersistency ( mem_t mem );
private:
char *RAN_volume_ = "260535867639876.7:99f12172-4095-4ca0-aef6-74ca5e6da9b9";
kdsa_vol_handle_t RAN_handle_;
double ioTime;
};
......
......@@ -55,8 +55,8 @@ int Topology::IONodesPerFile ( char* filename, int *nodesList ) {
int Topology::ListOfMemoryTiers ( mem_t* memList ) {
memList[0] = DDR;
memList[1] = NAM;
memList[2] = NVR;
memList[1] = NVR;
memList[2] = NAM;
return 3;
}
......
MPICXX = mpicxx
MPI_CFLAGS = -g -Wno-write-strings -O3 -I./ -I./architectures -I./architectures/$(ARCHI)
MPI_CFLAGS = -g -Wno-write-strings -O3 -I./ -I./architectures -I./architectures/$(ARCHI) -I/soft/RAN/kdsa_api/current/include/
MPI_CFLAGS += -DDBG -DCOLOR #-DTIMING
MPI_LDFLAGS = -L/$(HOME)/install/$(ARCHI)/lib
MPI_LDFLAGS = -L/$(HOME)/install/$(ARCHI)/lib -L/soft/RAN/kdsa_api/current/lib64/
TP_INCLUDE = architectures/tp_topology_interface.hpp \
architectures/tp_memory_interface.hpp \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment