Skip to content

GitLab

Commit 1d30b3e0 authored by Francois Tessier's avatar Francois Tessier
Browse files

Rewrite the header files for mem and net abstractions. Add a prototype to implement data uncaching

parent e1246397
Hide whitespace changes
Inline Side-by-side
Showing with 37 additions and 341 deletions
architectures/linux-sles_12-x86_64/tp_memory.hpp
View file @ 1d30b3e0
......@@ -3,190 +3,44 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>
#include <hbwmalloc.h>
#include "tp_utils.hpp"
#include "tp_memory_interface.hpp"
#include <hbwmalloc.h>
class Memory: public iMemory {
public:
Memory ( );
~Memory ( );
/**********************/
/* |-- Allocation */
/**********************/
void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm ) {
int rank;
this->mem_ = mem;
this->masterRank_ = masterRank;
MPI_Comm_dup( comm, &this->comm_ );
this->buffSize_ = buffSize;
strncpy ( this->fileName_, fileName, sizeof(fileName) );
MPI_Comm_rank ( this->comm_, &rank );
switch ( this->mem_ )
{
case DDR:
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Allocate memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
this->buffer_ = malloc ( this->buffSize_ );
MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
}
else
MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
MPI_Win_fence (0, this->RMAWin_ );
break;
case HBM:
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Allocate memory on HBM (%s:%d)\n", __FILE__, __LINE__ );
this->buffer_ = hbw_malloc ( this->buffSize_ );
MPI_Win_create ( this->buffer_, this->buffSize_, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
}
else
MPI_Win_create ( NULL, 0, 1, MPI_INFO_NULL, this->comm_, &this->RMAWin_ );
MPI_Win_fence (0, this->RMAWin_ );
break;
default:
printMsg ( ERROR, "Unable to allocate memory (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
void memFree ( ) {
int rank;
MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
switch ( this->mem_ )
{
case DDR:
MPI_Win_free ( &this->RMAWin_ );
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Free memory on DDR (%s:%d)\n", __FILE__, __LINE__ );
free ( this->buffer_ );
}
break;
case HBM:
MPI_Win_free ( &this->RMAWin_ );
if ( this->masterRank_ ) {
printMsg ( DEBUG, "Free memory on HBM (%s:%d)\n", __FILE__, __LINE__ );
hbw_free ( this->buffer_ );
}
break;
default:
printMsg ( ERROR, "Unable to free memory (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
void memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* fileName, MPI_Comm comm );
void memFree ( );
/**********************/
/* |-- I/O */
/**********************/
int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
int err;
MPI_Status status;
switch ( this->mem_ )
{
case DDR:
case HBM:
err = MPI_Put ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
break;
default:
printMsg ( ERROR, "Error while writing data (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
return err;
}
int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank ) {
int err;
MPI_Status status;
switch ( this->mem_ )
{
case DDR:
case HBM:
err = MPI_Get ( srcBuffer, srcSize, MPI_BYTE, destRank, offset, srcSize, MPI_BYTE, this->RMAWin_ );
break;
default:
printMsg ( ERROR, "Error while reading data (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
return err;
}
int memFlush ( ) {
int err;
MPI_Status status;
switch ( this->mem_ )
{
case DDR:
case HBM:
MPI_Win_fence ( 0, this->RMAWin_ );
break;
default:
printMsg ( ERROR, "Error while flushing data (mem = %s)\n", this->memName () );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
return err;
}
int memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank );
int memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int destRank );
int memFlush ( );
int memUncache ( );
/**********************/
/* |-- Utils */
/**********************/
char* memName ( ) {
switch ( this->mem_ )
{
case DDR:
return "DDR";
break;
case HBM:
return "HBM";
break;
case SSD:
return "SSD";
break;
case HDD:
return "HDD";
break;
default:
printMsg ( ERROR, "Wrong memory type!\n" );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
}
mem_t memType ( char* name ) {
if ( ! strcmp ( "DDR", name ) ) return DDR;
if ( ! strcmp ( "HBM", name ) ) return HBM;
if ( ! strcmp ( "SSD", name ) ) return SSD;
if ( ! strcmp ( "HDD", name ) ) return HDD;
printMsg ( ERROR, "Wrong memory name!\n" );
MPI_Abort ( MPI_COMM_WORLD, -1 );
}
char* memName ( );
mem_t memType ( char* name );
/************************/
/* |-- Characteristics */
/************************/
int64_t memBandwidth ( ) {
return 0;
}
int64_t memLatency ( ) {
return 0;
}
int64_t memCapacity ( ) {
return 0;
}
int64_t memBandwidth ( );
int64_t memLatency ( );
int64_t memCapacity ( );
bool memPersistency ( );
};
#endif // TP_MEMORY_H
......
architectures/linux-sles_12-x86_64/tp_network.hpp
View file @ 1d30b3e0
#ifndef CRAY_CX40_THETA_LUSTRE_H
#define CRAY_CX40_THETA_LUSTRE_H
#ifndef TP_NETWORK_H
#define TP_NETWORK_H
#include <stdio.h>
#include <stdlib.h>
#include <sched.h>
#include <mpi.h>
#include <pmi.h>
#include "tp_network_interface.hpp"
#include <pmi.h>
extern "C" {
#include <lustre/lustreapi.h>
......@@ -20,195 +22,35 @@ public:
/**********************/
/* |-- Nodes */
/**********************/
int IONodeId () {
return 0;
}
int BridgeNodeId () {
return 0;
}
int ComputeNodeId () {
return 0;
}
int ProcessPerNode () {
int ppn, err;
err = PMI_Get_numpes_on_smp ( &ppn );
if ( err != PMI_SUCCESS ) {
fprintf (stderr, "[ERROR] PMI_Get_numpes_on_smp failed!\n");
MPI_Abort (MPI_COMM_WORLD, -1);
}
return ppn;
}
/*
* 7 LNET nodes per OST
*/
int IONodesPerFile ( char* filename, int *nodesList ) {
int err, stripeCount, nLnets, i, idx, oid, l;
char fgrId [20];
int *ssuId, *ostId, *lnets;
struct find_param param = { 0 };
int ssu2fgr [] = { 0, 0, 0, 0,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5,
6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7,
8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9};
err = llapi_getstripe ( filename, &param );
if ( err )
fprintf ( stdout, "[ERROR] llapi_getstripe\n");
stripeCount = (&param)->fp_lmd->lmd_lmm.lmm_stripe_count;
nLnets = stripeCount * LNETS_PER_OST;
ssuId = (int *) malloc ( stripeCount * sizeof ( int ) );
ostId = (int *) malloc ( stripeCount * sizeof ( int ) );
// /!\ Hypothesis : OSS id == SNX - 4
for ( i = 0; i < stripeCount; i++ ) {
idx = (&param)->fp_lmd->lmd_lmm.lmm_objects[i].l_ost_idx;
ssuId[i] = idx + 4;
lnets = (int *) malloc ( LNETS_PER_OST * sizeof ( int ) );
snprintf ( fgrId, 20, "o2ib100%d", ssu2fgr[ ssuId[i] ] );
FgrToLnets ( fgrId, lnets );
for ( l = 0; l < LNETS_PER_OST; l++ )
nodesList [ i * LNETS_PER_OST + l ] = lnets [ l ];
free ( lnets );
}
return nLnets;
}
int IONodeId ();
int BridgeNodeId ();
int ComputeNodeId ();
int ProcessPerNode ();
int IONodesPerFile ( char* filename, int *nodesList );
/**********************/
/* |-- Network */
/**********************/
/*
* X : Rank 3 (Optics cables between groups : 12.5 Gbps)
* Y : Rank 2 (Copper cables between 6 backplanes : 14 Gbps)
* Z : Rank 1 (Copper cables between 16 aries routers in a backplane :14 Gbps)
* nid : Node identifier
*
* http://www.nersc.gov/users/computational-systems/edison/configuration/interconnect/
*
*/
int NetworkDimensions () {
return 4;
}
int NetworkDimensions ();
/* |---- Coordinates */
void RankToCoordinates ( int rank, int* coord ) {
pmi_mesh_coord_t xyz;
int nid;
/* Hypothesis : PMI_rank == MPI_rank */
PMI_Get_nid(rank, &nid);
PMI_Get_meshcoord((pmi_nid_t) nid, &xyz);
coord[0] = xyz.mesh_x;
coord[1] = xyz.mesh_y;
coord[2] = xyz.mesh_z;
coord[3] = nid;
coord[4] = sched_getcpu();
}
void IONodeCoordinates ( int* coord ) {
}
void RankToCoordinates ( int rank, int* coord );
void IONodeCoordinates ( int* coord );
/* |---- Distance */
int DistanceToIONode ( int srcRank ) {
return 0;
}
int DistanceToIONode ( int srcRank );
/* Minimal distance between two ranks considering a dragonfly network */
int DistanceBetweenRanks ( int srcRank, int destRank ) {
int dim = NetworkDimensions(), d;
int srcCoord[dim], destCoord[dim];
int distance;
RankToCoordinates ( srcRank, srcCoord );
RankToCoordinates ( destRank, destCoord );
distance = 0;
for ( d = 0; d < dim; d++ ) {
if ( srcCoord[d] != destCoord[d] )
distance++;
}
return distance;
}
int DistanceBetweenRanks ( int srcRank, int destRank );
/* |---- Routes */
int RouteToIONode ( int srcRank, int* path ) {
return 0;
}
int RouteBetweenRanks ( int srcRank, int destRank, int* path ) {
return 0;
}
int RouteToIONode ( int srcRank, int* path );
int RouteBetweenRanks ( int srcRank, int destRank, int* path );
/* |---- Links */
void LinksList ( int* linksList ) {
}
void LinksList ( int* linksList );
private:
void FgrToLnets ( char *fgr_id, int *lnet ) {
int count = 0;
FILE *fp;
char fline[100];
char *lnet_list, *item;
fp = fopen("/etc/lnet/routes.conf", "r");
if ( fp == NULL ) {
fprintf ( stdout, "[ERROR] Error while opening routes.conf file!\n" );
return;
}
while ( fgets ( fline, 100, fp ) != NULL ) {
const char *c = strstr ( fline, fgr_id );
if ( c != NULL ) {
const char *b1 = strstr ( fline, "[" ) + 1;
const char *b2 = strstr ( fline, "]" );
lnet_list = ( char * ) malloc ( sizeof ( char ) * ( b2 - b1 + 1 ) );
strncpy ( lnet_list, b1, b2 - b1 );
item = strtok ( lnet_list, "," );
while ( item ) {
lnet [ count ] = atoi ( item );
item = strtok ( 0, "," );
count++;
}
}
count = 0;
}
fclose ( fp );
return;
}
void FgrToLnets ( char *fgr_id, int *lnet );
};
#endif // CRAY_CX40_THETA_LUSTRE_H
#endif // TP_NETWORK_H
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment