Commit 88130b03 authored by Francois Tessier's avatar Francois Tessier

Function to print the distance matrix of nodes on BG/Q

parent 0c730fae
...@@ -12,4 +12,6 @@ examples/S3D-IO/run/*.out ...@@ -12,4 +12,6 @@ examples/S3D-IO/run/*.out
examples/S3D-IO/run/writeperf.log examples/S3D-IO/run/writeperf.log
examples/S3D-IO/data/* examples/S3D-IO/data/*
examples/S3D-IO/run/s3d_f90.x examples/S3D-IO/run/s3d_f90.x
examples/Fortran/simple_array examples/Fortran/simple_array
\ No newline at end of file examples/1D-Array/main_mpiio
examples/1D-Array/main_tapioca
\ No newline at end of file
...@@ -118,49 +118,57 @@ void Topology::LinksList ( int* list ) { ...@@ -118,49 +118,57 @@ void Topology::LinksList ( int* list ) {
void Topology::PrintNodesAdjencyList ( ) { void Topology::PrintNodesAdjencyList ( ) {
int rankCoords[5]; int *worldRanks, *worldDistIO;
int *worldCoords;
MPI_Comm nodeComm, masterComm; MPI_Comm nodeComm, masterComm;
int size, rank, nodeRank, masterRank, masterSize, i, color = 0; int size, rank, nodeRank, nodeSize, masterRank, masterSize, i, color = 0;
int src, dest; int src, dest, distanceIO, resultlen;
char name[MPI_MAX_PROCESSOR_NAME];
MPI_Comm_size (MPI_COMM_WORLD, &size); MPI_Comm_size (MPI_COMM_WORLD, &size);
MPI_Comm_rank (MPI_COMM_WORLD, &rank); MPI_Comm_rank (MPI_COMM_WORLD, &rank);
this->RankToCoordinates ( rank, rankCoords );
for ( i = 0; i < 5; i++ ) MPI_Get_processor_name( name, &resultlen );
color += rankCoords[i] * pow (10.0, (double)i);
color = rank / this->ProcessPerNode ();
MPI_Comm_split ( MPI_COMM_WORLD, color, rank, &nodeComm ); MPI_Comm_split ( MPI_COMM_WORLD, color, rank, &nodeComm );
MPI_Comm_rank ( nodeComm, &nodeRank ); MPI_Comm_rank ( nodeComm, &nodeRank );
MPI_Comm_size ( nodeComm, &nodeSize );
MPI_Comm_split ( MPI_COMM_WORLD, nodeRank == 0, rank, &masterComm ); MPI_Comm_split ( MPI_COMM_WORLD, nodeRank == 0, rank, &masterComm );
MPI_Comm_rank ( masterComm, &masterRank ); MPI_Comm_rank ( masterComm, &masterRank );
MPI_Comm_size ( masterComm, &masterSize ); MPI_Comm_size ( masterComm, &masterSize );
worldCoords = (int *) malloc ( masterSize * 5 * sizeof ( int ) ); worldRanks = (int *) malloc ( masterSize * sizeof ( int ) );
worldDistIO = (int *) malloc ( masterSize * sizeof ( int ) );
MPI_Gather ( rankCoords, 5, MPI_INT, worldCoords, 5, MPI_INT, 0, masterComm ); distanceIO = this->DistanceToIONode ( 0 );
if ( masterRank == 0 ) { MPI_Gather ( &rank, 1, MPI_INT, worldRanks, 1, MPI_INT, 0, masterComm );
MPI_Gather ( &distanceIO, 1, MPI_INT, worldDistIO, 1, MPI_INT, 0, masterComm );
if ( masterRank == 0 && masterSize == ( size / this->ProcessPerNode () ) ) {
fprintf ( stdout, " N " );
for ( src = 0; src < masterSize; src++ )
fprintf ( stdout, "%3d ", src );
fprintf ( stdout, " IO\n" );
for ( src = 0; src < masterSize; src++ ) { for ( src = 0; src < masterSize; src++ ) {
fprintf ( stdout, "%3d ", src );
for ( i = 0; i < 5; i++ )
fprintf ( stdout, "%d", worldCoords [ src * 5 + i ] );
fprintf ( stdout, ": " );
for ( dest = 0; dest < masterSize; dest++ ) { for ( dest = 0; dest < masterSize; dest++ ) {
if ( DistanceBetweenRanks ( src, dest ) == 1 ) { fprintf ( stdout, "%3d ", DistanceBetweenRanks ( worldRanks [ src ], worldRanks [ dest ] ) );
for ( i = 0; i < 5; i++ )
fprintf ( stdout, "%d", worldCoords [ dest * 5 + i ] );
fprintf ( stdout, " " );
}
} }
fprintf ( stdout, "\n" );
fprintf ( stdout, "%3d\n", worldDistIO [ src ] );
} }
fprintf ( stdout, " IO " );
for ( src = 0; src < masterSize; src++ ) {
fprintf ( stdout, "%3d ", worldDistIO [ src ] );
}
fprintf ( stdout, "%3d\n", 0 );
} }
return; return;
......
...@@ -48,6 +48,8 @@ int main (int argc, char * argv[]) ...@@ -48,6 +48,8 @@ int main (int argc, char * argv[])
mycolor = 42; mycolor = 42;
mykey = world_myrank; mykey = world_myrank;
tp.topology.PrintNodesAdjencyList ();
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_size(sub_comm, &sub_numtasks);
MPI_Comm_rank(sub_comm, &sub_myrank); MPI_Comm_rank(sub_comm, &sub_myrank);
...@@ -118,7 +120,7 @@ int main (int argc, char * argv[]) ...@@ -118,7 +120,7 @@ int main (int argc, char * argv[])
if (0 == world_myrank) { if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024); io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n", fprintf (stdout, BLUE "[TIMING]" RESET " Write I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000); io_bw, (double)tot_size/(1024*1024), max_time * 1000);
} }
MPI_Barrier (MPI_COMM_WORLD); MPI_Barrier (MPI_COMM_WORLD);
...@@ -156,7 +158,7 @@ int main (int argc, char * argv[]) ...@@ -156,7 +158,7 @@ int main (int argc, char * argv[])
if (0 == world_myrank) { if (0 == world_myrank) {
io_bw = (double)tot_size / max_time / (1024 * 1024); io_bw = (double)tot_size / max_time / (1024 * 1024);
fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n", fprintf (stdout, BLUE "[TIMING]" RESET " Read I/O bandwidth: %.2f MBps (%.2f MB in %.2f ms)\n",
io_bw, (double)tot_size/(1024*1024), max_time * 1000); io_bw, (double)tot_size/(1024*1024), max_time * 1000);
} }
MPI_Barrier (MPI_COMM_WORLD); MPI_Barrier (MPI_COMM_WORLD);
...@@ -169,8 +171,8 @@ int main (int argc, char * argv[]) ...@@ -169,8 +171,8 @@ int main (int argc, char * argv[])
for (uint64_t i = 0; i< num_particles; i++) { for (uint64_t i = 0; i< num_particles; i++) {
if (xx[i] != xx_r[i]) if (xx[i] != xx_r[i])
{ {
fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i); fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i);
MPI_Abort (MPI_COMM_WORLD, -1); MPI_Abort (MPI_COMM_WORLD, -1);
} }
} }
......
#!/bin/bash #!/bin/bash
NODES=8 NODES=32
PPN=16 PPN=16
NPROCS=$((NODES*PPN)) NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug" TARGET="/projects/visualization/ftessier/debug"
DDT="/soft/debuggers/ddt-7.0-2017-02-16/bin/ddt --connect"
cd $HOME/TAPIOCA/examples/1D-Array cd $HOME/TAPIOCA/examples/1D-Array
export TAPIOCA_DEVNULL=false export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=4 export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=4194304 export TAPIOCA_BUFFERSIZE=4194304
export TAPIOCA_PIPELINING=true export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=true export TAPIOCA_REELECTAGGR=true
...@@ -34,6 +35,7 @@ function updateSettings() ...@@ -34,6 +35,7 @@ function updateSettings()
SETTINGS="$SETTINGS GPFSMPIO_BALANCECONTIG=1" SETTINGS="$SETTINGS GPFSMPIO_BALANCECONTIG=1"
SETTINGS="$SETTINGS GPFSMPIO_P2PCONTIG=1" SETTINGS="$SETTINGS GPFSMPIO_P2PCONTIG=1"
#SETTINGS="$SETTINGS MPICH_MPIIO_HINTS=*:bg_nodes_pset=$TAPIOCA_NBAGGR" #SETTINGS="$SETTINGS MPICH_MPIIO_HINTS=*:bg_nodes_pset=$TAPIOCA_NBAGGR"
echo $SETTINGS
} }
function outputFile () function outputFile ()
...@@ -52,8 +54,10 @@ do ...@@ -52,8 +54,10 @@ do
updateSettings updateSettings
rm $TARGET/* rm $TARGET/*
outputFile "TAPIOCA" #outputFile "TAPIOCA"
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./main_tapioca $PARTICLES >> $OUTPUT runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./main_tapioca $PARTICLES
#$DDT --processes=$NPROCS : ./main_tapioca $PARTICLES
# >> $OUTPUT
sleep 5 sleep 5
# rm $TARGET/* # rm $TARGET/*
......
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
Tapioca::Tapioca () Tapioca::Tapioca ()
{ {
this->SetDefaultValues (); this->reElectAggr_ = true;
this->ParseEnvVariables (); this->electedAggr_ = false;
} }
...@@ -20,6 +20,9 @@ void Tapioca::Init (int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset, ...@@ -20,6 +20,9 @@ void Tapioca::Init (int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset,
startInitTime = MPI_Wtime(); startInitTime = MPI_Wtime();
#endif #endif
this->SetDefaultValues ();
this->ParseEnvVariables ();
this->nChunks_ = nChunks; this->nChunks_ = nChunks;
this->chunksIndexMatching.resize (this->nChunks_); this->chunksIndexMatching.resize (this->nChunks_);
...@@ -171,8 +174,6 @@ void Tapioca::SetDefaultValues () ...@@ -171,8 +174,6 @@ void Tapioca::SetDefaultValues ()
this->nAggr_ = 8; this->nAggr_ = 8;
this->bufferSize_ = 16777216; this->bufferSize_ = 16777216;
this->nBuffers_ = 2; this->nBuffers_ = 2;
this->reElectAggr_ = true;
this->electedAggr_ = false;
this->amAnAggr_ = false; this->amAnAggr_ = false;
this->commSplit_ = true; this->commSplit_ = true;
this->currentRound_ = 0; this->currentRound_ = 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment