From ef46bc18f0b3577d2afe70bce6b6a31cadbc134b Mon Sep 17 00:00:00 2001 From: Francois Tessier Date: Mon, 30 Oct 2017 21:00:27 +0000 Subject: [PATCH] Global declaration of tiers of memory for aggregation and output file in HACC-IO --- examples/HACC-IO/miniHACC-AoS-R.cpp | 15 +++++---- examples/HACC-IO/miniHACC-AoS-W.cpp | 2 +- examples/HACC-IO/miniHACC-AoS.cpp | 38 ++++++++++++---------- examples/HACC-IO/run_check_haswell.sh | 45 ++++++++++++++++++++------- 4 files changed, 64 insertions(+), 36 deletions(-) diff --git a/examples/HACC-IO/miniHACC-AoS-R.cpp b/examples/HACC-IO/miniHACC-AoS-R.cpp index abf60d7..656b12e 100644 --- a/examples/HACC-IO/miniHACC-AoS-R.cpp +++ b/examples/HACC-IO/miniHACC-AoS-R.cpp @@ -11,6 +11,9 @@ #define BLUE "\x1b[34m" #define RESET "\x1b[0m" +mem_t memAggr = NVR; +mem_t memTarg = HDD; + int main (int argc, char * argv[]) { int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; @@ -29,9 +32,8 @@ int main (int argc, char * argv[]) MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &world_numtasks); MPI_Comm_rank(MPI_COMM_WORLD, &world_myrank); - + mycolor = tp.topology.BridgeNodeId (); - //mycolor = 42; mykey = world_myrank; MPI_Comm_split (MPI_COMM_WORLD, mycolor, mykey, &sub_comm); @@ -43,7 +45,7 @@ int main (int argc, char * argv[]) file_id = 0; if ( argv[2] != NULL ) file_id = atoi ( argv[2] ); - + #ifdef BGQ snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id); #elif XC40 @@ -140,8 +142,8 @@ int main (int argc, char * argv[]) } tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); - tp.setAggregationTier (2, DDR, "/scratch/tmp"); - tp.setTargetTier (HDD, file_size, output); + tp.setAggregationTier (2, memAggr, "/scratch/tmp"); + tp.setTargetTier (memTarg, file_size, output); /*****************/ start_time = MPI_Wtime(); @@ -197,7 +199,8 @@ int main (int argc, char * argv[]) || (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i]) || (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i])) { - fprintf (stdout, RED "[ERROR]" RESET " Wrong value for particle %d\n", i); + fprintf (stdout, RED "[ERROR]" RESET "[%02d] Wrong value for particle %d (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %lld, %u )\n", + world_myrank, i, xx_r[i], yy_r[i], zz_r[i], vx_r[i], vy_r[i], vz_r[i], phi_r[i], pid_r[i], mask_r[i] ); MPI_Abort (MPI_COMM_WORLD, -1); } } diff --git a/examples/HACC-IO/miniHACC-AoS-W.cpp b/examples/HACC-IO/miniHACC-AoS-W.cpp index bec77b1..d12633a 100644 --- a/examples/HACC-IO/miniHACC-AoS-W.cpp +++ b/examples/HACC-IO/miniHACC-AoS-W.cpp @@ -45,7 +45,7 @@ int main (int argc, char * argv[]) file_id = atoi ( argv[2] ); #ifdef BGQ - snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); + snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-AOS-%08d-%d.dat", mycolor, file_id); #elif XC40 snprintf (output, 100, "/lus/theta-fs0/projects/Performance/ftessier/HACC/HACC-AOS-%08d-%d.dat", mycolor, file_id); #else diff --git a/examples/HACC-IO/miniHACC-AoS.cpp b/examples/HACC-IO/miniHACC-AoS.cpp index 9728bd0..594a32b 100644 --- a/examples/HACC-IO/miniHACC-AoS.cpp +++ b/examples/HACC-IO/miniHACC-AoS.cpp @@ -11,6 +11,9 @@ #define BLUE "\x1b[34m" #define RESET "\x1b[0m" +mem_t memAggr = NVR; +mem_t memTarg = HDD; + int main (int argc, char * argv[]) { int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; @@ -22,7 +25,7 @@ int main (int argc, char * argv[]) MPI_Comm sub_comm; MPI_Status status; char output[100]; - Tapioca tp; + Tapioca tp, tp2; int64_t chunkCount[9], chunkOffset[9]; int chunkSize[9]; @@ -122,8 +125,8 @@ int main (int argc, char * argv[]) } tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); - tp.setAggregationTier (5, NVR, "/scratch/tmp"); - tp.setTargetTier (HDD, file_size, output); + tp.setAggregationTier (2, memAggr, "/scratch/tmp-write"); + tp.setTargetTier (memTarg, file_size, output); /*****************/ start_time = MPI_Wtime(); @@ -190,9 +193,9 @@ int main (int argc, char * argv[]) /*****************/ /* INIT TAPIOCA */ /*****************/ - tp.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); - tp.setAggregationTier (4, NVR, "/scratch/tmp"); - tp.setTargetTier (HDD, file_size, output); + tp2.Init (chunkCount, chunkSize, chunkOffset, 9, hdr, sub_comm); + tp2.setAggregationTier (2, memAggr, "/scratch/tmp-read"); + tp2.setTargetTier (memTarg, file_size, output); /*****************/ start_time = MPI_Wtime(); @@ -202,31 +205,31 @@ int main (int argc, char * argv[]) offset = scan_size * particle_size; - tp.Read (offset, xx_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, xx_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, yy_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, yy_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, zz_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, zz_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, vx_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, vx_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, vy_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, vy_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, vz_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, vz_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, phi_r, num_particles, MPI_FLOAT, &status); + tp2.Read (offset, phi_r, num_particles, MPI_FLOAT, &status); offset += num_particles * sizeof(float); - tp.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status); + tp2.Read (offset, pid_r, num_particles, MPI_LONG_LONG, &status); offset += num_particles * sizeof(int64_t); - tp.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); + tp2.Read (offset, mask_r, num_particles, MPI_UNSIGNED_SHORT, &status); end_time = MPI_Wtime(); tot_time = end_time - start_time; @@ -238,7 +241,7 @@ int main (int argc, char * argv[]) io_bw, (double)tot_size/(1024*1024), max_time * 1000); } - tp.Finalize (); + tp2.Finalize (); /*****************/ /* VERIFICATION */ @@ -248,7 +251,8 @@ int main (int argc, char * argv[]) || (vx[i] != vx_r[i]) || (vy[i] != vy_r[i]) || (vz[i] != vz_r[i]) || (phi[i] != phi_r[i])|| (pid[i] != pid_r[i]) || (mask[i] != mask_r[i])) { - fprintf (stdout, RED "[ERROR]" RESET "[%03d] Wrong value for particle %d\n", world_myrank, i); + fprintf (stdout, RED "[ERROR]" RESET "[%02d] Wrong value for particle %d (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %lld, %u )\n", + world_myrank, i, xx_r[i], yy_r[i], zz_r[i], vx_r[i], vy_r[i], vz_r[i], phi_r[i], pid_r[i], mask_r[i] ); MPI_Abort (MPI_COMM_WORLD, -1); } } diff --git a/examples/HACC-IO/run_check_haswell.sh b/examples/HACC-IO/run_check_haswell.sh index 8778075..5d564fc 100755 --- a/examples/HACC-IO/run_check_haswell.sh +++ b/examples/HACC-IO/run_check_haswell.sh @@ -13,6 +13,7 @@ export TAPIOCA_COMMSPLIT=true export TAPIOCA_STRATEGY=TOPOLOGY_AWARE export TAPIOCA_NBAGGR=4 export TAPIOCA_BUFFERSIZE=16777216 +#export TAPIOCA_BUFFERSIZE=33554432 export TAPIOCA_PIPELINING=true export PAMID_VERBOSE=1 @@ -28,28 +29,48 @@ function outputFile () printenv | egrep "TAPIOCA_" +PARTICLES=200000 #rm $TARGET/* #mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS 15000 # ls -l $TARGET/ # sleep 5 +#mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R $PARTICLES rm $TARGET/* -mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS 100000 +mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-MPIIO $PARTICLES sleep 3 -#ls -l /projects/visualization/ftessier/debug/ -#md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat -# sleep 60 -# rm $TARGET/* -# mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-MPIIO 60000 -# sleep 3 -# ls -l /projects/visualization/ftessier/debug/ -# md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat +ls -l /projects/visualization/ftessier/debug/ +md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat +echo +echo "---------------------------------------------" +echo +rm $TARGET/* +mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-W $PARTICLES +sleep 3 +ls -l /projects/visualization/ftessier/debug/ +md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat +echo +echo "---------------------------------------------" +echo +sleep 3 +mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R $PARTICLES +echo +echo "---------------------------------------------" +echo +sleep 3 +rm $TARGET/* +mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS $PARTICLES +ls -l /projects/visualization/ftessier/debug/ +md5sum /projects/visualization/ftessier/debug/HACC-AOS-00000000-0.dat +# echo +# echo "---------------------------------------------" +# echo +# sleep 3 +# mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS $PARTICLES +# sleep 1 #mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-SoA-R 15000 # sleep 5 -echo -echo "---------------------------------------------" -echo #mpirun -f $COBALT_NODEFILE -n $NPROCS ./miniHACC-AoS-R 15000 # for run in {0..9} -- GitLab