Skip to content

GitLab

Commit b888c8b5 authored by ftessier's avatar ftessier
Browse files

Tests to run on BG/Q to figure out the bugs on XC40

parent c9603c9c
Hide whitespace changes
Inline Side-by-side
Showing with 39 additions and 36 deletions
examples/HACC-IO/Makefile.xc40
View file @ b888c8b5
MPICXX = CC
MPI_CFLAGS = -g -O3 -I$(HOME)/install/include/ -DXC40 -DDEBUG
MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi
MPI_LDFLAGS = -L$(HOME)/install/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
all: miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
......
examples/HACC-IO/miniHACC-AoS.cpp
View file @ b888c8b5
......@@ -31,7 +31,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId ();
//mycolor = tp.topology.BridgeNodeId ();
mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
......@@ -137,31 +138,31 @@ int main (int argc, char * argv[])
offset = scan_size * particle_size;
tp.Write (file_handle, offset, xx, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, xx, num_particles, MPI_FLOAT, &status, "xx");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, yy, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, yy, num_particles, MPI_FLOAT, &status, "yy");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, zz, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, zz, num_particles, MPI_FLOAT, &status, "zz");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, vx, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, vx, num_particles, MPI_FLOAT, &status, "vx");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, vy, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, vy, num_particles, MPI_FLOAT, &status, "vy");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, vz, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, vz, num_particles, MPI_FLOAT, &status, "vz");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, phi, num_particles, MPI_FLOAT, &status);
tp.Write (file_handle, offset, phi, num_particles, MPI_FLOAT, &status, "phi");
offset += num_particles * sizeof(float);
tp.Write (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status);
tp.Write (file_handle, offset, pid, num_particles, MPI_LONG_LONG, &status, "pid");
offset += num_particles * sizeof(int64_t);
tp.Write (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status);
tp.Write (file_handle, offset, mask, num_particles, MPI_UNSIGNED_SHORT, &status, "mask");
MPI_File_close (&file_handle);
......
examples/HACC-IO/miniHACC-SoA.cpp
View file @ b888c8b5
......@@ -14,7 +14,7 @@
int main (int argc, char * argv[])
{
int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id;
int64_t num_particles = 25005;
int64_t num_particles = 100000;
int64_t sub_particles, tot_particles, particle_size, file_size, tot_size;
int64_t scan_size = 0, offset, hdr = 0;
double start_time, end_time, tot_time, max_time;
......@@ -31,8 +31,8 @@ int main (int argc, char * argv[])
MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks);
MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank);
mycolor = tp.topology.BridgeNodeId ();
//mycolor = 42;
//mycolor = tp.topology.BridgeNodeId ();
mycolor = 42;
mykey = world_myrank;
MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm);
......
examples/HACC-IO/run_check_xc40.sh
View file @ b888c8b5
#!/bin/bash
VARS=""
NODES=10
VARS="-e MPICH_RMA_OVER_DMAPP=1"
NODES=8
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
......@@ -31,7 +31,7 @@ function updateSettings()
rm $TARGET/*
updateSettings
aprun $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS
aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS 1
#########################
# Array of Structures
#########################
......
make.inc/Makefile.xc40
View file @ b888c8b5
......@@ -3,6 +3,6 @@ MPICXX = CC
MPI_CFLAGS = -g -O3 -I./topology/ -I./topology/include/
MPI_CFLAGS += -DXC40 -DDEBUG #-DTIMING
MPI_LDFLAGS += -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
MPI_LDFLAGS = -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
INSTALL_PATH = $(HOME)/install
tapioca.cpp
View file @ b888c8b5
......@@ -162,7 +162,7 @@ void Tapioca::Finalize ()
int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
int count, MPI_Datatype datatype, MPI_Status *status, int64_t bufOffset)
int count, MPI_Datatype datatype, MPI_Status *status, char* var, int64_t bufOffset)
{
int retval, i, c, targetRoundIdx, targetAggrIdx, targetGlobAggr;
int typeSize, targetAggr, win, buffer;
......@@ -193,15 +193,15 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
*/
while ( this->roundsIds[targetRoundIdx] > this->currentRound_ ) {
for ( i = 0; i < this->nAggr_ ; i++ ) {
this->GlobalFence ();
this->GlobalFence (var);
this->roundCounter_++;
}
if ( this->amAnAggr_ ) {
if (iRequest != NULL)
MPI_Wait ( &iRequest, &iStatus );
// this->Push (fileHandle, status);
this->iPush (fileHandle, &iRequest);
// if (iRequest != NULL)
// MPI_Wait ( &iRequest, &iStatus );
this->Push (fileHandle, status);
//this->iPush (fileHandle, &iRequest);
}
this->currentRound_++;
......@@ -237,14 +237,14 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
*/
if ( this->currentDataSize_ == this->rankDataSize_) {
while ( this->roundCounter_ < this->totalRounds_) {
this->GlobalFence ();
this->GlobalFence (var);
if ( (this->roundCounter_ % this->nAggr_) == 0 ) {
if ( this->amAnAggr_ ) {
if (iRequest != NULL)
MPI_Wait ( &iRequest, &iStatus );
// this->Push (fileHandle, status);
this->iPush (fileHandle, &iRequest);
// if (iRequest != NULL)
// MPI_Wait ( &iRequest, &iStatus );
this->Push (fileHandle, status);
//this->iPush (fileHandle, &iRequest);
}
this->currentRound_++;
}
......@@ -255,14 +255,14 @@ int Tapioca::Write (MPI_File fileHandle, MPI_Offset offset, void *buf,
if ( multipleRounds ) {
retval = this->Write (fileHandle, offset + subChunkDataSize, buf,
chunkDataSize - subChunkDataSize, MPI_BYTE, status, subChunkDataSize);
chunkDataSize - subChunkDataSize, MPI_BYTE, status, var, subChunkDataSize);
}
else {
this->nCommit_ ++;
}
if (iRequest != NULL)
MPI_Wait ( &iRequest, &iStatus );
// if (iRequest != NULL)
// MPI_Wait ( &iRequest, &iStatus );
return retval;
}
......@@ -273,6 +273,7 @@ void Tapioca::Push (MPI_File fileHandle, MPI_Status *status)
int64_t offset, dataSize;
int win, buffer;
fprintf (stdout, "[DEBUG] Enter Push\n");
buffer = this->currentRound_ % NBUFFERS;
if ( this->amAnAggr_ ) {
......@@ -359,11 +360,12 @@ void Tapioca::iPush (MPI_File fileHandle, MPI_Request *request)
}
void Tapioca::GlobalFence ()
void Tapioca::GlobalFence (char* var)
{
int buffer;
buffer = this->currentRound_ % NBUFFERS;
fprintf (stdout, "[INFO] %d %s buffer = %d\n", this->commRank_, var, buffer);
switch (buffer)
{
......@@ -1067,7 +1069,7 @@ void Tapioca::HandleMPIError (int retval)
int resultlen;
if (retval != MPI_SUCCESS) {
MPI_Error_string(retval, msg, &resultlen);
fprintf(stderr, "%s\n", msg);
fprintf(stderr, "[ERR] %s\n", msg);
}
#endif
......
tapioca.hpp
View file @ b888c8b5
......@@ -57,7 +57,7 @@ class Tapioca
int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm);
int Write (MPI_File fileHandle, MPI_Offset offset,
void *buf, int count, MPI_Datatype datatype,
MPI_Status *status, int64_t bufOffset = 0);
MPI_Status *status, char* var, int64_t bufOffset = 0);
void ReadInitialize (char *filename, int64_t *chunkCount, int *chunkSize, int64_t *chunkOffset,
int nChunks, int64_t offset, MEMORY_LAYOUT layout, MPI_Comm comm);
......@@ -90,7 +90,7 @@ class Tapioca
void InitAggregators ();
void Push (MPI_File fileHandle, MPI_Status *status);
void iPush (MPI_File fileHandle, MPI_Request *request);
void GlobalFence ();
void GlobalFence (char* var);
/***********************/
/* PLACEMENT */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment