Commit b888c8b5 authored by ftessier's avatar ftessier
Browse files

Tests to run on BG/Q to figure out the bugs on XC40

parent c9603c9c
MPICXX = CC MPICXX = CC
MPI_CFLAGS = -g -O3 -I$(HOME)/install/include/ -DXC40 -DDEBUG MPI_CFLAGS = -g -O3 -I$(HOME)/install/include/ -DXC40 -DDEBUG
MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
......
...@@ -31,7 +31,8 @@ int main (int argc, char * argv[]) ...@@ -31,7 +31,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId (); //mycolor = tp.topology.BridgeNodeId ();
mycolor = 42;
mykey = world_myrank; mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
...@@ -137,31 +138,31 @@ int main (int argc, char * argv[]) ...@@ -137,31 +138,31 @@ int main (int argc, char * argv[])
offset = scan_size * particle_size; offset = scan_size * particle_size;
tp.Write (file_handle, offset, xx, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, xx, num_particles, MPI_FLOAT, &status, "xx");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, yy, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, yy, num_particles, MPI_FLOAT, &status, "yy");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, zz, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, zz, num_particles, MPI_FLOAT, &status, "zz");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, vx, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, vx, num_particles, MPI_FLOAT, &status, "vx");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, vy, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, vy, num_particles, MPI_FLOAT, &status, "vy");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, vz, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, vz, num_particles, MPI_FLOAT, &status, "vz");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, phi, num_particles, MPI_FLOAT, &status); tp.Write (file_handle, offset, phi, num_particles, MPI_FLOAT, &status, "phi");
offset += num_particles * sizeof(float); offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status); tp.Write (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status, "pid");
offset += num_particles * sizeof(int64_t); offset += num_particles * sizeof(int64_t);
tp.Write (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status); tp.Write (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status, "mask");
MPI_File_close (&file_handle); MPI_File_close (&file_handle);
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
int main (int argc, char * argv[]) int main (int argc, char * argv[])
{ {
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles = 25005; int64_t num_particles = 100000;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0; int64_t scan_size = 0, offset, hdr = 0;
double start_time, end_time, tot_time, max_time; double start_time, end_time, tot_time, max_time;
...@@ -31,8 +31,8 @@ int main (int argc, char * argv[]) ...@@ -31,8 +31,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId (); //mycolor = tp.topology.BridgeNodeId ();
//mycolor = 42; mycolor = 42;
mykey = world_myrank; mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
......
#!/bin/bash #!/bin/bash
VARS="" VARS="-e MPICH_RMA_OVER_DMAPP=1"
NODES=10 NODES=8
PPN=16 PPN=16
NPROCS=$((NODES*PPN)) NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC" TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
...@@ -31,7 +31,7 @@ function updateSettings() ...@@ -31,7 +31,7 @@ function updateSettings()
rm $TARGET/* rm $TARGET/*
updateSettings updateSettings
aprun $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS 1
######################### #########################
# Array of Structures # Array of Structures
######################### #########################
......
...@@ -3,6 +3,6 @@ MPICXX = CC ...@@ -3,6 +3,6 @@ MPICXX = CC
MPI_CFLAGS = -g -O3 -I./topology/ -I./topology/include/ MPI_CFLAGS = -g -O3 -I./topology/ -I./topology/include/
MPI_CFLAGS += -DXC40 -DDEBUG #-DTIMING MPI_CFLAGS += -DXC40 -DDEBUG #-DTIMING
MPI_LDFLAGS += -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive MPI_LDFLAGS = -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
INSTALL_PATH = $(HOME)/install INSTALL_PATH = $(HOME)/install
...@@ -162,7 +162,7 @@ void Tapioca::Finalize () ...@@ -162,7 +162,7 @@ void Tapioca::Finalize ()
int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf, int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
int count, MPI_Datatype datatype, MPI_Status *status, int64_t bufOffset) int count, MPI_Datatype datatype, MPI_Status *status, char* var, int64_t bufOffset)
{ {
int retval, i, c, targetRoundIdx, targetAggrIdx, targetGlobAggr; int retval, i, c, targetRoundIdx, targetAggrIdx, targetGlobAggr;
int typeSize, targetAggr, win, buffer; int typeSize, targetAggr, win, buffer;
...@@ -193,15 +193,15 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf, ...@@ -193,15 +193,15 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
*/ */
while ( this->roundsIds[targetRoundIdx] > this->currentRound_ ) { while ( this->roundsIds[targetRoundIdx] > this->currentRound_ ) {
for ( i = 0; i < this->nAggr_ ; i++ ) { for ( i = 0; i < this->nAggr_ ; i++ ) {
this->GlobalFence (); this->GlobalFence (var);
this->roundCounter_++; this->roundCounter_++;
} }
if ( this->amAnAggr_ ) { if ( this->amAnAggr_ ) {
if (iRequest != NULL) // if (iRequest != NULL)
MPI_Wait ( &iRequest, &iStatus ); // MPI_Wait ( &iRequest, &iStatus );
// this->Push (fileHandle, status); this->Push (fileHandle, status);
this->iPush (fileHandle, &iRequest); //this->iPush (fileHandle, &iRequest);
} }
this->currentRound_++; this->currentRound_++;
...@@ -237,14 +237,14 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf, ...@@ -237,14 +237,14 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
*/ */
if ( this->currentDataSize_ == this->rankDataSize_) { if ( this->currentDataSize_ == this->rankDataSize_) {
while ( this->roundCounter_ < this->totalRounds_) { while ( this->roundCounter_ < this->totalRounds_) {
this->GlobalFence (); this->GlobalFence (var);
if ( (this->roundCounter_ % this->nAggr_) == 0 ) { if ( (this->roundCounter_ % this->nAggr_) == 0 ) {
if ( this->amAnAggr_ ) { if ( this->amAnAggr_ ) {
if (iRequest != NULL) // if (iRequest != NULL)
MPI_Wait ( &iRequest, &iStatus ); // MPI_Wait ( &iRequest, &iStatus );
// this->Push (fileHandle, status); this->Push (fileHandle, status);
this->iPush (fileHandle, &iRequest); //this->iPush (fileHandle, &iRequest);
} }
this->currentRound_++; this->currentRound_++;
} }
...@@ -255,14 +255,14 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf, ...@@ -255,14 +255,14 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
if ( multipleRounds ) { if ( multipleRounds ) {
retval = this->Write (fileHandle, offset + subChunkDataSize, buf, retval = this->Write (fileHandle, offset + subChunkDataSize, buf,
chunkDataSize - subChunkDataSize, MPI_BYTE, status, subChunkDataSize); chunkDataSize - subChunkDataSize, MPI_BYTE, status, var, subChunkDataSize);
} }
else { else {
this->nCommit_ ++; this->nCommit_ ++;
} }
if (iRequest != NULL) // if (iRequest != NULL)
MPI_Wait ( &iRequest, &iStatus ); // MPI_Wait ( &iRequest, &iStatus );
return retval; return retval;
} }
...@@ -273,6 +273,7 @@ void Tapioca::Push (MPI_File fileHandle, MPI_Status *status) ...@@ -273,6 +273,7 @@ void Tapioca::Push (MPI_File fileHandle, MPI_Status *status)
int64_t offset, dataSize; int64_t offset, dataSize;
int win, buffer; int win, buffer;
fprintf (stdout, "[DEBUG] Enter Push\n");
buffer = this->currentRound_ % NBUFFERS; buffer = this->currentRound_ % NBUFFERS;
if ( this->amAnAggr_ ) { if ( this->amAnAggr_ ) {
...@@ -359,11 +360,12 @@ void Tapioca::iPush (MPI_File fileHandle, MPI_Request *request) ...@@ -359,11 +360,12 @@ void Tapioca::iPush (MPI_File fileHandle, MPI_Request *request)
} }
void Tapioca::GlobalFence () void Tapioca::GlobalFence (char* var)
{ {
int buffer; int buffer;
buffer = this->currentRound_ % NBUFFERS; buffer = this->currentRound_ % NBUFFERS;
fprintf (stdout, "[INFO] %d %s buffer = %d\n", this->commRank_, var, buffer);
switch (buffer) switch (buffer)
{ {
...@@ -1067,7 +1069,7 @@ void Tapioca::HandleMPIError (int retval) ...@@ -1067,7 +1069,7 @@ void Tapioca::HandleMPIError (int retval)
int resultlen; int resultlen;
if (retval != MPI_SUCCESS) { if (retval != MPI_SUCCESS) {
MPI_Error_string(retval, msg, &resultlen); MPI_Error_string(retval, msg, &resultlen);
fprintf(stderr, "%s\n", msg); fprintf(stderr, "[ERR] %s\n", msg);
} }
#endif #endif
......
...@@ -57,7 +57,7 @@ class Tapioca ...@@ -57,7 +57,7 @@ class Tapioca
int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm); int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm);
int Write (MPI_File fileHandle, MPI_Offset offset, int Write (MPI_File fileHandle, MPI_Offset offset,
void *buf, int count, MPI_Datatype datatype, void *buf, int count, MPI_Datatype datatype,
MPI_Status *status, int64_t bufOffset = 0); MPI_Status *status, char* var, int64_t bufOffset = 0);
void ReadInitialize (char *filename, int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset, void ReadInitialize (char *filename, int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset,
int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm); int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm);
...@@ -90,7 +90,7 @@ class Tapioca ...@@ -90,7 +90,7 @@ class Tapioca
void InitAggregators (); void InitAggregators ();
void Push (MPI_File fileHandle, MPI_Status *status); void Push (MPI_File fileHandle, MPI_Status *status);
void iPush (MPI_File fileHandle, MPI_Request *request); void iPush (MPI_File fileHandle, MPI_Request *request);
void GlobalFence (); void GlobalFence (char* var);
/***********************/ /***********************/
/* PLACEMENT */ /* PLACEMENT */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment