Commit 88130b03 authored by Francois Tessier's avatar Francois Tessier

Function to print the distance matrix of nodes on BG/Q

parent 0c730fae
......@@ -12,4 +12,6 @@ examples/S3D-IO/run/*.out
examples/S3D-IO/run/writeperf.log
examples/S3D-IO/data/*
examples/S3D-IO/run/s3d_f90.x
examples/Fortran/simple_array
\ No newline at end of file
examples/Fortran/simple_array
examples/1D-Array/main_mpiio
examples/1D-Array/main_tapioca
\ No newline at end of file
......@@ -118,49 +118,57 @@ void Topology::LinksList ( int* list ) {
void Topology::PrintNodesAdjencyList ( ) {
int rankCoords[5];
int *worldCoords;
int *worldRanks, *worldDistIO;
MPI_Comm nodeComm, masterComm;
int size, rank, nodeRank, masterRank, masterSize, i, color = 0;
int src, dest;
int size, rank, nodeRank, nodeSize, masterRank, masterSize, i, color = 0;
int src, dest, distanceIO, resultlen;
char name[MPI_MAX_PROCESSOR_NAME];
MPI_Comm_size (MPI_COMM_WORLD, &size);
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
this->RankToCoordinates ( rank, rankCoords );
for ( i = 0; i < 5; i++ )
color += rankCoords[i] * pow (10.0, (double)i);
MPI_Get_processor_name( name, &resultlen );
color = rank / this->ProcessPerNode ();
MPI_Comm_split ( MPI_COMM_WORLD, color, rank, &nodeComm );
MPI_Comm_rank ( nodeComm, &nodeRank );
MPI_Comm_size ( nodeComm, &nodeSize );
MPI_Comm_split ( MPI_COMM_WORLD, nodeRank == 0, rank, &masterComm );
MPI_Comm_rank ( masterComm, &masterRank );
MPI_Comm_size ( masterComm, &masterSize );
worldCoords = (int *) malloc ( masterSize * 5 * sizeof ( int ) );
MPI_Gather ( rankCoords, 5, MPI_INT, worldCoords, 5, MPI_INT, 0, masterComm );
worldRanks = (int *) malloc ( masterSize * sizeof ( int ) );
worldDistIO = (int *) malloc ( masterSize * sizeof ( int ) );
distanceIO = this->DistanceToIONode ( 0 );
if ( masterRank == 0 ) {
MPI_Gather ( &rank, 1, MPI_INT, worldRanks, 1, MPI_INT, 0, masterComm );
MPI_Gather ( &distanceIO, 1, MPI_INT, worldDistIO, 1, MPI_INT, 0, masterComm );
if ( masterRank == 0 && masterSize == ( size / this->ProcessPerNode () ) ) {
fprintf ( stdout, " N " );
for ( src = 0; src < masterSize; src++ )
fprintf ( stdout, "%3d ", src );
fprintf ( stdout, " IO\n" );
for ( src = 0; src < masterSize; src++ ) {
for ( i = 0; i < 5; i++ )
fprintf ( stdout, "%d", worldCoords [ src * 5 + i ] );
fprintf ( stdout, ": " );
fprintf ( stdout, "%3d ", src );
for ( dest = 0; dest < masterSize; dest++ ) {
if ( DistanceBetweenRanks ( src, dest ) == 1 ) {
for ( i = 0; i < 5; i++ )
fprintf ( stdout, "%d", worldCoords [ dest * 5 + i ] );
fprintf ( stdout, " " );
}
fprintf ( stdout, "%3d ", DistanceBetweenRanks ( worldRanks [ src ], worldRanks [ dest ] ) );
}
fprintf ( stdout, "\n" );
fprintf ( stdout, "%3d\n", worldDistIO [ src ] );
}
fprintf ( stdout, " IO " );
for ( src = 0; src < masterSize; src++ ) {
fprintf ( stdout, "%3d ", worldDistIO [ src ] );
}
fprintf ( stdout, "%3d\n", 0 );
}
return;
......
......@@ -48,6 +48,8 @@ int main (int argc, char * argv[])
mycolor = 42;
mykey = world_myrank;
tp.topology.PrintNodesAdjencyList ();
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank);
......@@ -118,7 +120,7 @@ int main (int argc, char * argv[])
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
MPI_Barrier (MPI_COMM_WORLD);
......@@ -156,7 +158,7 @@ int main (int argc, char * argv[])
if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
io_bw, (double)tot_size/(1024*1024), max_time * 1000);
}
MPI_Barrier (MPI_COMM_WORLD);
......@@ -169,8 +171,8 @@ int main (int argc, char * argv[])
for (uint64_t i = 0; i< num_particles; i++) {
if (xx[i] != xx_r[i])
{
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1);
}
}
......
#!/bin/bash
NODES=8
NODES=32
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
DDT="/soft/debuggers/ddt-7.0-2017-02-16/bin/ddt --connect"
cd $HOME/TAPIOCA/examples/1D-Array
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=4
export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=4194304
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=true
......@@ -34,6 +35,7 @@ function updateSettings()
SETTINGS="$SETTINGS GPFSMPIO_BALANCECONTIG=1"
SETTINGS="$SETTINGS GPFSMPIO_P2PCONTIG=1"
#SETTINGS="$SETTINGS MPICH_MPIIO_HINTS=*:bg_nodes_pset=$TAPIOCA_NBAGGR"
echo $SETTINGS
}
function outputFile ()
......@@ -52,8 +54,10 @@ do
updateSettings
rm $TARGET/*
outputFile "TAPIOCA"
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./main_tapioca $PARTICLES >> $OUTPUT
#outputFile "TAPIOCA"
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./main_tapioca $PARTICLES
#$DDT --processes=$NPROCS : ./main_tapioca $PARTICLES
# >> $OUTPUT
sleep 5
# rm $TARGET/*
......
......@@ -2,8 +2,8 @@
Tapioca::Tapioca ()
{
this->SetDefaultValues ();
this->ParseEnvVariables ();
this->reElectAggr_ = true;
this->electedAggr_ = false;
}
......@@ -20,6 +20,9 @@ void Tapioca::Init (int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset,
startInitTime = MPI_Wtime();
#endif
this->SetDefaultValues ();
this->ParseEnvVariables ();
this->nChunks_ = nChunks;
this->chunksIndexMatching.resize (this->nChunks_);
......@@ -171,8 +174,6 @@ void Tapioca::SetDefaultValues ()
this->nAggr_ = 8;
this->bufferSize_ = 16777216;
this->nBuffers_ = 2;
this->reElectAggr_ = true;
this->electedAggr_ = false;
this->amAnAggr_ = false;
this->commSplit_ = true;
this->currentRound_ = 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment