Skip to content

GitLab

Commit a75d012d authored by Francois Tessier's avatar Francois Tessier
Browse files

First implementation of memory abstraction for aggregation buffers. Ongoing XP.

parent 1c90dfb0
Hide whitespace changes
Inline Side-by-side
Makefile
View file @ a75d012d
......@@ -12,7 +12,7 @@ libtapioca.a:$(OBJ_FILES)
$(MPICXX) $(MPI_CFLAGS) -c $? -o $@ $(MPI_LDFLAGS)
install:
cp -r tapioca.hpp $(TP_INCLUDE) $(INSTALL_PATH)/include
cp -r tapioca.hpp tp_utils.hpp $(TP_INCLUDE) $(INSTALL_PATH)/include
cp libtapioca.a $(INSTALL_PATH)/lib
clean:
......
architectures/linux-sles_12-x86_64/tp_memory.hpp
View file @ a75d012d
......@@ -14,12 +14,12 @@ public:
/**********************/
/* |-- Allocation */
/**********************/
void memAlloc ( int64_t buffSize, mem_t mem, int masterRank, char* fileName, MPI_Comm comm ) {
void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) {
int rank;
this->mem_ = mem;
this->masterRank_ = masterRank;
this->comm_ = comm;
MPI_Comm_dup( comm, &this->comm_ );
this->buffSize_ = buffSize;
strcpy ( this->fileName_, fileName);
......@@ -28,7 +28,7 @@ public:
switch ( this->mem_ )
{
case DDR:
if ( rank == this->masterRank_ ) {
if ( this->masterRank_ ) {
this->buffer_ = malloc ( this->buffSize_ );
MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
}
......@@ -51,7 +51,7 @@ public:
{
case DDR:
MPI_Win_free ( &this->RMAWin_ );
if ( rank == this->masterRank_ )
if ( this->masterRank_ )
free ( this->buffer_ );
break;
default:
......@@ -63,14 +63,14 @@ public:
/**********************/
/* |-- I/O */
/**********************/
int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset ) {
int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
int err;
MPI_Status status;
switch ( this->mem_ )
{
case DDR:
err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, 0, offset, srcSize, MPI_BYTE, this->RMAWin_ );
err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
break;
default:
printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
......@@ -80,14 +80,14 @@ public:
return err;
}
int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset ) {
int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
int err;
MPI_Status status;
switch ( this->mem_ )
{
case DDR:
err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, 0, offset, srcSize, MPI_BYTE, this->RMAWin_ );
err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
break;
default:
printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
......
architectures/tp_memory_interface.hpp
View file @ a75d012d
......@@ -11,37 +11,39 @@ public:
/************************/
/* |-- Allocation */
/************************/
virtual void memAlloc ( int64_t buffSize, mem_t mem, int masterRank, char* fileName, MPI_Comm comm );
virtual void memFree ( );
virtual void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) = 0;
virtual void memFree ( ) = 0;
/************************/
/* |-- I/O */
/************************/
//virtual int memWrite ( iMemory srcMem, int64_t srcSize, int64_t offset );
//virtual int memRead ( iMemory srcMem, int64_t srcSize, int64_t offset );
virtual int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset );
virtual int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset );
virtual int memFlush ( );
virtual int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) = 0;
virtual int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) = 0;
virtual int memFlush ( ) = 0;
/************************/
/* |-- Utils */
/************************/
virtual char* memName ( );
virtual mem_t memType ( char* name );
virtual char* memName ( ) = 0;
virtual mem_t memType ( char* name ) = 0;
/************************/
/* |-- Characteristics */
/************************/
virtual int64_t memBandwidth ( );
virtual int64_t memLatency ( );
virtual int64_t memCapacity ( );
virtual int64_t memBandwidth ( ) = 0;
virtual int64_t memLatency ( ) = 0;
virtual int64_t memCapacity ( ) = 0;
/* Temporary */
void *buffer_;
protected:
mem_t mem_;
int masterRank_;
bool masterRank_;
MPI_Comm comm_;
void *buffer_;
int64_t buffSize_;
MPI_Win RMAWin_;
......
examples/HACC-IO/run_check_xc40.sh
View file @ a75d012d
#!/bin/bash
NODES=1024
NODES=32
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT="/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT=48
STRIPE_COUNT=4
STRIPE_SIZE=8388608
AGGR_MULT=1
......@@ -14,7 +14,7 @@ cd $HOME/install/$ARCHI/bin/
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=192
export TAPIOCA_NBAGGR=4
export TAPIOCA_BUFFERSIZE=16777216
function setLustreFS ()
......@@ -41,29 +41,23 @@ function updateSettings()
function outputFile ()
{
OUTPUT="$HOME/PUB-IPDPS18/dat/aggr_count"
OUTPUT="$HOME/PUB-IPDPS18/dat/mem_abstraction"
OUTPUT="${OUTPUT}/HACC_XC40_${BIN}_${NODES}_${PPN}"
OUTPUT="${OUTPUT}_${TAPIOCA_STRATEGY}_${TAPIOCA_NBAGGR}_${TAPIOCA_BUFFERSIZE}_${PARTICLES}"
}
for run in {0..9}
for run in {0..2}
do
for PARTICLES in 25000
do
for AGGR in 48 96 144 192
do
AGGR_MULT=$((AGGR/STRIPE_COUNT))
echo "cray_cb_nodes_multiplier=$AGGR_MULT"
export TAPIOCA_NBAGGR=$AGGR
updateSettings
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS >> $OUTPUT
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
sleep 5
done
updateSettings
for BIN in miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS >> $OUTPUT
aprun $SETTINGS -n $NPROCS -N $PPN ./$BIN $PARTICLES >> $OUTPUT
sleep 5
done
done
done
......
tapioca.cpp
View file @ a75d012d
......@@ -20,15 +20,9 @@ void Tapioca::Finalize ()
this->commDataSize_ = 0;
MPI_Win_free (&this->RMAWin1);
MPI_Win_free (&this->RMAWin2);
this->memBuffer0.memFree ();
this->memBuffer1.memFree ();
MPI_Comm_free (&this->subComm_);
if ( this->amAnAggr_ ) {
free (this->buffer1);
free (this->buffer2);
}
}
......@@ -41,10 +35,10 @@ void Tapioca::GlobalFence ()
switch (buffer)
{
case 0:
MPI_Win_fence (0, this->RMAWin1);
this->memBuffer0.memFlush ();
break;
case 1:
MPI_Win_fence (0, this->RMAWin2);
this->memBuffer1.memFlush ();
break;
}
......@@ -420,26 +414,8 @@ void Tapioca::InitAggregators ()
{
int aggr, retval;
if ( this->amAnAggr_ ) {
this->buffer1 = malloc (this->bufferSize_);
this->buffer2 = malloc (this->bufferSize_);
retval = MPI_Win_create (this->buffer1, this->bufferSize_, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin1);
this->HandleMPIError (retval);
MPI_Win_create (this->buffer2, this->bufferSize_, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin2);
this->HandleMPIError (retval);
}
else {
retval = MPI_Win_create (NULL, 0, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin1);
this->HandleMPIError (retval);
retval = MPI_Win_create (NULL, 0, 1, MPI_INFO_NULL, this->subComm_, &this->RMAWin2);
this->HandleMPIError (retval);
}
retval = MPI_Win_fence (0, this->RMAWin1);
this->HandleMPIError (retval);
retval = MPI_Win_fence (0, this->RMAWin2);
this->HandleMPIError (retval);
this->memBuffer0.memAlloc ( this->bufferSize_, DDR, this->amAnAggr_, NULL, this->subComm_ );
this->memBuffer1.memAlloc ( this->bufferSize_, DDR, this->amAnAggr_, NULL, this->subComm_ );
#ifdef DEBUG
if (this->commRank_ == MASTER) {
......
tapioca.hpp
View file @ a75d012d
......@@ -19,6 +19,7 @@
#include "mpi.h"
#include "tp_network.hpp"
#include "tp_memory.hpp"
enum MAPPING_STRATEGY
{
......@@ -157,11 +158,9 @@ class Tapioca
std::vector<int> dataSize;
std::vector< std::vector<int> > chunksIndexMatching;
void *buffer1;
void *buffer2;
MPI_Win RMAWin1;
MPI_Win RMAWin2;
Memory memBuffer0;
Memory memBuffer1;
//Memory memTarget;
/* AGGREGATOR */
bool amAnAggr_;
......
tp_read.cpp
View file @ a75d012d
......@@ -130,14 +130,10 @@ int Tapioca::Read (MPI_File fileHandle, MPI_Offset offset, void *buf,
switch (buffer)
{
case 0:
retval = MPI_Get (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin1);
this->HandleMPIError (retval);
this->memBuffer0.memRead ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
break;
case 1:
retval = MPI_Get (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin2);
this->HandleMPIError (retval);
this->memBuffer1.memRead ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
break;
}
......@@ -198,11 +194,11 @@ void Tapioca::Pull (MPI_File fileHandle, MPI_Request *request)
switch (buffer)
{
case 0:
MPI_File_iread_at (fileHandle, offset, buffer1, dataSize, MPI_BYTE, request);
MPI_File_iread_at (fileHandle, offset, this->memBuffer0.buffer_, dataSize, MPI_BYTE, request);
MPI_Wait ( request, &status );
break;
case 1:
MPI_File_iread_at (fileHandle, offset, buffer2, dataSize, MPI_BYTE, request);
MPI_File_iread_at (fileHandle, offset, this->memBuffer1.buffer_, dataSize, MPI_BYTE, request);
MPI_Wait ( request, &status );
break;
}
......
tp_write.cpp
View file @ a75d012d
......@@ -126,14 +126,10 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
switch (buffer)
{
case 0:
retval = MPI_Put (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin1);
this->HandleMPIError (retval);
this->memBuffer0.memWrite ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
break;
case 1:
retval = MPI_Put (static_cast<char*>(buf) + bufOffset, subChunkDataSize, MPI_BYTE,
targetAggr, winOffset, subChunkDataSize, MPI_BYTE, this->RMAWin2);
this->HandleMPIError (retval);
this->memBuffer1.memWrite ( static_cast<char*>(buf) + bufOffset, subChunkDataSize, winOffset, targetAggr );
break;
}
......@@ -196,15 +192,15 @@ void Tapioca::Push (MPI_File fileHandle, MPI_Request *request)
{
case 0:
if ( this->writeDevNull_ )
MPI_File_iwrite_at (this->devNullFileHandle_, 0, buffer1, dataSize, MPI_BYTE, request);
MPI_File_iwrite_at (this->devNullFileHandle_, 0, this->memBuffer0.buffer_, dataSize, MPI_BYTE, request);
else
MPI_File_iwrite_at (fileHandle, offset, buffer1, dataSize, MPI_BYTE, request);
MPI_File_iwrite_at (fileHandle, offset, this->memBuffer0.buffer_, dataSize, MPI_BYTE, request);
break;
case 1:
if ( this->writeDevNull_ )
MPI_File_iwrite_at (this->devNullFileHandle_, 0, buffer2, dataSize, MPI_BYTE, request);
MPI_File_iwrite_at (this->devNullFileHandle_, 0, this->memBuffer1.buffer_, dataSize, MPI_BYTE, request);
else
MPI_File_iwrite_at (fileHandle, offset, buffer2, dataSize, MPI_BYTE, request);
MPI_File_iwrite_at (fileHandle, offset, this->memBuffer1.buffer_, dataSize, MPI_BYTE, request);
break;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment