From 109885b69a5cfe97783e11f8e81c1fd7e84d0bc0 Mon Sep 17 00:00:00 2001 From: ftessier Date: Mon, 24 Apr 2017 12:13:30 +0000 Subject: [PATCH] User-defined number of particles per rank for HACC-IO --- examples/HACC-IO/miniHACC-AoS-MPIIO.cpp | 4 ++- examples/HACC-IO/miniHACC-AoS.cpp | 8 +++-- examples/HACC-IO/miniHACC-SoA-MPIIO.cpp | 4 ++- examples/HACC-IO/miniHACC-SoA.cpp | 4 ++- examples/HACC-IO/run_check_xc40.sh | 48 ++++++++++++++++++------- 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/examples/HACC-IO/miniHACC-AoS-MPIIO.cpp b/examples/HACC-IO/miniHACC-AoS-MPIIO.cpp index f7ff9b5..cba04b1 100644 --- a/examples/HACC-IO/miniHACC-AoS-MPIIO.cpp +++ b/examples/HACC-IO/miniHACC-AoS-MPIIO.cpp @@ -14,7 +14,7 @@ int main (int argc, char * argv[]) { int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id; - int64_t num_particles = 25000; + int64_t num_particles; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t scan_size = 0, offset; double start_time, end_time, tot_time, max_time; @@ -36,6 +36,8 @@ int main (int argc, char * argv[]) MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_rank(sub_comm, &sub_myrank); + num_particles = atoi ( argv[1] ); + file_id = 0; if ( argv[1] != NULL ) file_id = atoi ( argv[1] ); diff --git a/examples/HACC-IO/miniHACC-AoS.cpp b/examples/HACC-IO/miniHACC-AoS.cpp index 63929aa..9b22ee4 100644 --- a/examples/HACC-IO/miniHACC-AoS.cpp +++ b/examples/HACC-IO/miniHACC-AoS.cpp @@ -14,7 +14,7 @@ int main (int argc, char * argv[]) { int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; - int64_t num_particles = 25000; + int64_t num_particles; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t scan_size = 0, offset, hdr = 0; double start_time, end_time, tot_time, max_time; @@ -38,9 +38,11 @@ int main (int argc, char * argv[]) MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_rank(sub_comm, &sub_myrank); + num_particles = atoi ( argv[1] ); + file_id = 0; - if ( argv[1] != NULL ) - file_id = atoi ( argv[1] ); + if ( argv[2] != NULL ) + file_id = atoi ( argv[2] ); #ifdef BGQ snprintf (output, 100, "/projects/visualization/ftessier/debug/HACC-SOA-%08d-%d.dat", mycolor, file_id); diff --git a/examples/HACC-IO/miniHACC-SoA-MPIIO.cpp b/examples/HACC-IO/miniHACC-SoA-MPIIO.cpp index a404d28..a832569 100644 --- a/examples/HACC-IO/miniHACC-SoA-MPIIO.cpp +++ b/examples/HACC-IO/miniHACC-SoA-MPIIO.cpp @@ -14,7 +14,7 @@ int main (int argc, char * argv[]) { int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, file_id; - int64_t num_particles = 25000; + int64_t num_particles; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t scan_size = 0, offset; double start_time, end_time, tot_time, max_time; @@ -36,6 +36,8 @@ int main (int argc, char * argv[]) MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_rank(sub_comm, &sub_myrank); + num_particles = atoi ( argv[1] ); + file_id = 0; if ( argv[1] != NULL ) file_id = atoi ( argv[1] ); diff --git a/examples/HACC-IO/miniHACC-SoA.cpp b/examples/HACC-IO/miniHACC-SoA.cpp index 79f85f3..0bfd970 100644 --- a/examples/HACC-IO/miniHACC-SoA.cpp +++ b/examples/HACC-IO/miniHACC-SoA.cpp @@ -14,7 +14,7 @@ int main (int argc, char * argv[]) { int world_numtasks, world_myrank, mycolor, mykey, sub_numtasks, sub_myrank, i, file_id; - int64_t num_particles = 25000; + int64_t num_particles; int64_t sub_particles, tot_particles, particle_size, file_size, tot_size; int64_t scan_size = 0, offset, hdr = 0; double start_time, end_time, tot_time, max_time; @@ -38,6 +38,8 @@ int main (int argc, char * argv[]) MPI_Comm_size(sub_comm, &sub_numtasks); MPI_Comm_rank(sub_comm, &sub_myrank); + num_particles = atoi ( argv[1] ); + file_id = 0; if ( argv[1] != NULL ) file_id = atoi ( argv[1] ); diff --git a/examples/HACC-IO/run_check_xc40.sh b/examples/HACC-IO/run_check_xc40.sh index 54bbbd6..8a6c55e 100755 --- a/examples/HACC-IO/run_check_xc40.sh +++ b/examples/HACC-IO/run_check_xc40.sh @@ -1,22 +1,33 @@ #!/bin/bash -VARS="-e MPICH_RMA_OVER_DMAPP=1 -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1 -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=8" -NODES=64 +NODES=1024 PPN=16 NPROCS=$((NODES*PPN)) TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC" DDT="/soft/debuggers/forge/bin/ddt --connect" +STRIPE_COUNT=48 +STRIPE_SIZE=8388608 cd $HOME/TAPIOCA/examples/HACC-IO export TAPIOCA_DEVNULL=false export TAPIOCA_COMMSPLIT=true export TAPIOCA_STRATEGY=TOPOLOGY_AWARE -export TAPIOCA_NBAGGR=8 +export TAPIOCA_NBAGGR=64 #export TAPIOCA_BUFFERSIZE=2097152 #export TAPIOCA_BUFFERSIZE=4194304 #export TAPIOCA_BUFFERSIZE=8388608 export TAPIOCA_BUFFERSIZE=16777216 +#export TAPIOCA_BUFFERSIZE=33554432 + +VARS="-e MPICH_RMA_OVER_DMAPP=1 -e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1 -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=1" + +function setLustreFS () +{ + rm $TARGET/* + lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET + lfs getstripe $TARGET +} function updateSettings() { @@ -30,17 +41,28 @@ function updateSettings() SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1" } +for PARTICLES in 5000 15000 25000 35000 50000 +do + updateSettings + setLustreFS + + rm $TARGET/* + aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS $PARTICLES 1 + sleep 5 + + rm $TARGET/* + aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS-MPIIO $PARTICLES 1 + sleep 5 + + rm $TARGET/* + aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA $PARTICLES 1 + sleep 5 + + rm $TARGET/* + aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA-MPIIO $PARTICLES 1 + sleep 5 +done -rm $TARGET/* -updateSettings - -aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS 1 -sleep 5 -aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-AoS-MPIIO 1 -sleep 5 -aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA 1 -sleep 5 -aprun $VARS $SETTINGS -n $NPROCS -N $PPN ./miniHACC-SoA-MPIIO 1 ######################### # Array of Structures ######################### -- 2.24.1